src/app/core/database/pouchdb/synced-pouch-database.ts
Thrown by SyncedPouchDatabase.ensureSynced when the sync failed. The sync has already logged the failure: callers only need to tell the user.
No results matching.
Properties |
|
constructor(cause: unknown)
|
||||||
|
Parameters :
|
| Readonly Optional details |
Type : Record<string, unknown>
|
|
Inherited from
NamedError
|
|
Context for the report, kept out of the message so that it does not split the issue. |
import { PouchDatabase } from "./pouch-database";
import { toTypedDatabaseError } from "./database-exception";
import { Logging } from "../../logging/logging.service";
import { NamedError } from "../../logging/reported-errors";
import { KeycloakAuthService } from "../../session/auth/keycloak/keycloak-auth.service";
import { NgZone } from "@angular/core";
import { RemotePouchDatabase } from "./remote-pouch-database";
import {
debounceTime,
filter,
mergeMap,
takeUntil,
takeWhile,
} from "rxjs/operators";
import { SyncState } from "../../session/session-states/sync-state.enum";
import {
LoginStateSubject,
SyncStateSubject,
} from "../../session/session-type";
import { EMPTY, from, interval, merge, of } from "rxjs";
import { LoginState } from "../../session/session-states/login-state.enum";
import { NotAvailableOfflineError } from "../../session/not-available-offline.error";
import { AlertService } from "../../alerts/alert.service";
import { QueryOptions } from "../database";
import { PouchdbCorruptionRecoveryService } from "./pouchdb-corruption-recovery.service";
import { isDocumentWriteError } from "./pouchdb-errors";
import { ConnectivityError } from "#src/app/utils/backend-errors";
import { LAST_SYNC_KEY_PREFIX } from "#src/bootstrap-reset";
/**
* An alternative implementation of PouchDatabase that additionally
* provides functionality to sync with a remote CouchDB.
*/
export class SyncedPouchDatabase extends PouchDatabase {
get LAST_SYNC_KEY(): string | undefined {
if (!this.pouchDB?.name) {
return undefined;
}
return LAST_SYNC_KEY_PREFIX + this.pouchDB.name;
}
/**
* Destroy the local database.
* Its last completed sync is forgotten, so that a new database with the same name
* is not taken for local data that was lost (see DatabaseResolverService).
*/
override async destroy(): Promise<any> {
const lastSyncKey = this.LAST_SYNC_KEY;
const result = await super.destroy();
if (lastSyncKey) {
localStorage.removeItem(lastSyncKey);
}
return result;
}
POUCHDB_SYNC_BATCH_SIZE = 100;
SYNC_INTERVAL = 30000;
/**
* Abort a sync that makes no progress within this window (ms).
* Push writes have no per-request abort timeout, so a stale/half-open
* connection can leave the replication promise unsettled forever - which would
* keep syncState at STARTED and block liveSync from starting any further sync.
*
* The timer resets on every replication progress event, so this is an
* inactivity window per batch (see POUCHDB_SYNC_BATCH_SIZE) and not a budget
* for the total sync duration - a long initial sync of a large database keeps
* resetting the timer and is never cancelled while it makes progress.
* The window still has to cover the slowest legitimate gap between events:
* the checkpoint lookup and first `_changes` response before any batch
* arrives, and a single batch of documents with attachments over a poor
* connection. If that gap is exceeded no batch completes, so no checkpoint is
* written and every retry restarts from the same position - which is why this
* is set generously rather than close to a normal sync's duration.
*/
SYNC_STALL_TIMEOUT = 300000;
private readonly navigator: Navigator;
private readonly loginStateSubject: LoginStateSubject;
private readonly alertService?: AlertService;
private readonly corruptionRecovery?: PouchdbCorruptionRecoveryService;
private remoteDatabase: RemotePouchDatabase;
private syncState: SyncStateSubject = new SyncStateSubject();
/**
* Get the internal sync state subject for this database (not the global one).
* Useful for observing when this specific database's sync completes.
*/
get localSyncState(): SyncStateSubject {
return this.syncState;
}
constructor(
dbName: string,
authService: KeycloakAuthService,
globalSyncState: SyncStateSubject,
navigator: Navigator,
loginStateSubject: LoginStateSubject,
ngZone?: NgZone,
alertService?: AlertService,
corruptionRecovery?: PouchdbCorruptionRecoveryService,
) {
super(dbName, globalSyncState, ngZone);
this.navigator = navigator;
this.loginStateSubject = loginStateSubject;
this.alertService = alertService;
this.corruptionRecovery = corruptionRecovery;
this.remoteDatabase = new RemotePouchDatabase(
dbName,
authService,
undefined,
ngZone,
this.alertService,
);
this.logSyncContext();
this.syncState
.pipe(
takeUntil(this.destroy$),
filter((state) => state === SyncState.COMPLETED),
)
.subscribe(() => {
const lastSyncTime = new Date().toISOString();
localStorage.setItem(this.LAST_SYNC_KEY, lastSyncTime);
this.logSyncContext();
});
// forward sync state to global sync state (combining state from all synced databases)
this.syncState
.pipe(takeUntil(this.destroy$))
.subscribe((state: SyncState) => this.globalSyncState.next(state));
// Start live sync whenever the user is logged in.
// Note: if the user logged in offline (no Keycloak token), syncing will
// hit a 401 and the RemotePouchDatabase fetch interceptor will trigger a
// Keycloak login redirect. This is intentional — we want users to
// authenticate online when connectivity is available.
this.loginStateSubject
.pipe(
takeUntil(this.destroy$),
filter((state) => state === LoginState.LOGGED_IN),
)
.subscribe(() => this.liveSync());
}
/**
* Initializes the PouchDB with local indexeddb as well as a remote server connection for syncing.
* @param dbName local database name (for the current user)
* @param remoteDbName (optional) remote database name (if different from local browser database name)
*/
override init(dbName?: string, remoteDbName?: string) {
super.init(dbName ?? this.dbName, undefined, true);
// keep remote database on default name (e.g. "app" instead of "user_uuid-app")
this.remoteDatabase.init(remoteDbName, { trackLostPermissions: true });
}
/** doc counts of the most recent completed sync (for logging context) */
private lastSyncStats?: { pushed?: number; pulled?: number };
private async logSyncContext() {
const lastSyncTime = localStorage.getItem(this.LAST_SYNC_KEY);
// the context key includes the database name: every synced database (app,
// notifications, ...) writes its own block. With a shared key the last
// writer would overwrite the others, so a reported event could show the
// sync state of a different database than the one it is about.
Logging.addContext(`Aam Digital sync: ${this.dbName}`, {
db: this.dbName,
"last sync completed": lastSyncTime,
"last sync docs pushed": this.lastSyncStats?.pushed,
"last sync docs pulled": this.lastSyncStats?.pulled,
});
}
/**
* Execute a (one-time) sync between the local and server database.
*/
async sync(
options: PouchDB.Replication.SyncOptions = {},
): Promise<SyncResult> {
if (!this.navigator.onLine) {
Logging.debug("Not syncing because offline");
this.syncState.next(SyncState.UNSYNCED);
return {};
}
let localDb: PouchDB.Database;
let isFirstSync: boolean;
try {
localDb = await this.getPouchDBOnceReady();
isFirstSync = (await localDb.info()).doc_count === 0;
} catch (err) {
// e.g. a broken IndexedDB: logged like any failed sync (see ensureSynced)
const failure = toTypedDatabaseError(err);
await this.handleSyncFailure(failure, []);
throw failure;
}
if (isFirstSync) {
// On first sync there are no local docs, so skip lost-permission tracking & purge
this.remoteDatabase.trackLostPermissions = false;
}
this.syncState.next(SyncState.STARTED);
// Track the last batch of synced doc IDs for diagnostics on write failures
let lastSyncedDocIds: string[] = [];
// Run PouchDB sync/replication outside Angular zone to:
// - avoid wasted change-detection cycles for internal sync chatter
// - prevent expected internal rejections (e.g. transient 404s for
// not-yet-existing remote DBs) from being routed to Angular's
// ErrorHandler / Sentry. Outer .then/.catch below still handle them
// explicitly and re-enter the zone for state updates.
const createSyncHandler = () =>
localDb.sync(this.remoteDatabase.getPouchDB(), {
batch_size: this.POUCHDB_SYNC_BATCH_SIZE,
...options,
});
const syncHandler = this.ngZone
? this.ngZone.runOutsideAngular(createSyncHandler)
: createSyncHandler();
// Guard against a stalled sync (see SYNC_STALL_TIMEOUT): if replication makes
// no progress within the timeout, cancel it so the promise settles and
// liveSync can retry, instead of staying blocked in STARTED forever.
let stallTimer: ReturnType<typeof setTimeout>;
let rejectStalled: (err: any) => void;
const stallGuard = new Promise<never>((_, reject) => {
rejectStalled = reject;
});
const armStallTimer = () => {
clearTimeout(stallTimer);
stallTimer = setTimeout(() => {
Logging.debug(`sync stalled, cancelling to allow retry`, {
db: this.dbName,
});
// Reject BEFORE cancelling: PouchDB's replication thenable resolves
// (fires "complete") on cancel(), so cancelling first could let
// Promise.race take the success path and mark a stalled sync COMPLETED.
rejectStalled(new SyncStalledError());
syncHandler.cancel();
}, this.SYNC_STALL_TIMEOUT);
};
syncHandler.on("change", (info) => {
armStallTimer();
lastSyncedDocIds =
info?.change?.docs?.map((d) => d._id).filter(Boolean) ?? [];
});
syncHandler.on("active", () => armStallTimer());
syncHandler.on("paused", () => armStallTimer());
// per-doc rejections (e.g. 401/403 during push) do not fail the overall
// sync, but leave the rejected change in the local database only
const deniedDocs: DeniedDoc[] = [];
syncHandler.on("denied", (event: { doc?: DeniedDoc }) =>
deniedDocs.push(event?.doc ?? {}),
);
armStallTimer();
return Promise.race([syncHandler, stallGuard])
.then(async (res) => {
if (res) res["dbName"] = this.dbName; // add for debugging information
Logging.debug("sync completed", res);
this.lastSyncStats = {
pushed: (res as SyncResult)?.push?.docs_written,
pulled: (res as SyncResult)?.pull?.docs_written,
};
if (!isFirstSync) {
await this.purgeDocsWithLostPermissions();
}
this.syncState.next(SyncState.COMPLETED);
return res as SyncResult;
})
.catch(async (err) => {
// Handle 404 errors for notifications database (may not exist yet if no event was triggered)
if (this.isNotificationsDatabase() && err?.status === 404) {
Logging.debug(
"Notifications database does not exist yet on server - this may be expected",
err,
);
this.syncState.next(SyncState.COMPLETED);
return {};
}
// the replication fails with a copy of the error, which loses the class of a typed error
const failure = toTypedDatabaseError(err);
await this.handleSyncFailure(failure, lastSyncedDocIds);
throw failure;
})
.finally(() => {
clearTimeout(stallTimer);
if (isFirstSync) {
this.remoteDatabase.trackLostPermissions = true;
}
this.logDeniedDocs(deniedDocs);
});
}
/**
* Report the docs the server refused during one sync in a single warning,
* since a single push can be refused for hundreds of docs at once.
* Only counts and entity types, no document ids (see #4174).
*/
private logDeniedDocs(deniedDocs: DeniedDoc[]) {
if (deniedDocs.length === 0) {
return;
}
const ids = deniedDocs.map((doc) => doc.id).filter(Boolean);
Logging.warn("sync: server denied replication of documents", {
db: this.dbName,
denied: deniedDocs.length,
errors: [...new Set(deniedDocs.map((doc) => doc.name))],
entityTypes: [...new Set(ids.map((id) => id.split(":")[0]))],
});
}
/**
* Log a failed sync according to its cause (only once: callers like
* {@link ensureSynced} rely on this) and mark the sync state as failed.
*/
private async handleSyncFailure(err: any, lastSyncedDocIds: string[]) {
await this.logSyncFailure(err, lastSyncedDocIds);
this.syncState.next(SyncState.FAILED);
}
private async logSyncFailure(err: any, lastSyncedDocIds: string[]) {
const context = {
db: this.dbName,
...lastSyncedBatchContext(lastSyncedDocIds),
};
if (isDocumentWriteError(err)) {
Logging.warn(
`sync failed: document write error (possible oversized document)`,
context,
err,
);
return;
}
// reported (with the offer to reset the local data) by the recovery service
const reportedAsCorruption =
await this.corruptionRecovery?.handleKnownMultiTabCorruption(
err,
"sync failed: likely multi-tab IndexedDB corruption",
context,
this.pouchDB,
);
if (reportedAsCorruption) {
return;
}
// at debug if offline or the session expired (see `Logging`): retried later
Logging.warn(`sync failed`, { db: this.dbName }, err);
}
override async put(object: any, forceOverwrite = false): Promise<any> {
try {
return await super.put(object, forceOverwrite);
} catch (err) {
await this.corruptionRecovery?.handleKnownMultiTabCorruption(
err,
"put failed: likely multi-tab IndexedDB corruption",
{ db: this.dbName },
this.pouchDB,
);
throw err;
}
}
override query(
fun: string | ((doc: any, emit: any) => void),
options: QueryOptions,
): Promise<any> {
return super.query(fun, options).catch(async (err) => {
await this.corruptionRecovery?.handleKnownMultiTabCorruption(
err,
"query failed: likely multi-tab IndexedDB corruption",
{ db: this.dbName },
this.pouchDB,
);
throw err;
});
}
/**
* How many document ids to include as an example in the logs below, which can
* report on thousands of documents (the count is logged in full alongside).
*/
private readonly SAMPLE_IDS_LOGGED = 5;
/**
* Purge local documents for which the server reported lost permissions
* during the most recent sync's `_changes` calls.
*
* The server reports every document changed since this client's checkpoint
* that the user may not read - it has no knowledge of which of those the
* client actually holds locally. Most reported ids therefore do not exist
* locally at all and purging them is a no-op. Only an actual local deletion
* is logged with details, so that the log stays a signal for data being
* removed rather than a count of what the server reported. That is the
* expected outcome of a change of permissions, so it is not reported to
* remote monitoring; only a failed purge is.
*/
private async purgeDocsWithLostPermissions(): Promise<void> {
const reportedIds = this.remoteDatabase
.collectAndClearLostPermissions()
// design docs for indices are managed locally (and shouldn't be synced anyway)
.filter((id) => !id.startsWith("_design/"));
if (reportedIds.length === 0) {
return;
}
if (this.adapter === "idb") {
// a known limitation, not an event: the legacy "idb" adapter does not implement purge()
// (LocalPermissionEnforcerService destroys the local db instead where necessary)
Logging.debug(
"sync: cannot purge local docs on the legacy idb adapter, skipping",
{ db: this.dbName, reportedByServer: reportedIds.length },
);
return;
}
const purgedIds: string[] = [];
const failedIds: string[] = [];
let notPresentLocally = 0;
let firstError: unknown;
for (const _id of reportedIds) {
try {
if (await this.purge(_id)) {
purgedIds.push(_id);
} else {
notPresentLocally++;
}
} catch (err) {
failedIds.push(_id);
firstError ??= err;
}
}
const stats = {
db: this.dbName,
purged: purgedIds.length,
notPresentLocally,
failed: failedIds.length,
reportedByServer: reportedIds.length,
};
if (purgedIds.length > 0) {
// expected after a change of permissions: the user can no longer read these docs,
// so local changes to them could not have been synced anyway
Logging.debug(
"sync: purged local docs that the server reported as no longer permitted",
{
...stats,
sampleIds: purgedIds.slice(0, this.SAMPLE_IDS_LOGGED),
// the sync triggering this purge has replicated but not yet recorded its
// own timestamp, so this is the previously completed sync - i.e. how long
// this device had been out of sync *before* the sync that just succeeded.
previousSyncCompleted: localStorage.getItem(this.LAST_SYNC_KEY),
},
);
} else {
Logging.debug(
"sync: no local docs to purge for server-reported lost permissions",
stats,
);
}
if (failedIds.length > 0) {
// the local database keeps docs the user is no longer allowed to read
Logging.warn(
"sync: failed to purge local docs that the server reported as no longer permitted",
{ ...stats, sampleIds: failedIds.slice(0, this.SAMPLE_IDS_LOGGED) },
firstError,
);
}
}
/**
* Force a full re-check against the remote DB without deleting local data.
* Uses `checkpoint: false` once so PouchDB ignores previous checkpoints for this run.
*/
async resetSync(): Promise<void> {
Logging.debug(`triggering full re-sync for "${this.dbName}"`);
await this.sync({ checkpoint: false });
}
/**
* Ensure the database is synced with the remote server.
* Throws {@link NotAvailableOfflineError} if offline, or a {@link SyncFailedError}
* if the sync failed (which `sync()` has already logged).
* Otherwise triggers a one-time sync and resolves when complete.
*/
async ensureSynced(): Promise<void> {
if (!this.navigator.onLine) {
throw new NotAvailableOfflineError(
"Failed to ensure synced. Cannot sync database while offline.",
);
}
try {
await this.sync();
} catch (err) {
throw new SyncFailedError(err);
}
if (this.syncState.value === SyncState.UNSYNCED) {
throw new NotAvailableOfflineError(
"Failed to ensure synced. SyncState still reported as UNSYNCED.",
);
}
}
/**
* Continuous syncing in background.
*/
liveSyncEnabled: boolean;
liveSync() {
this.liveSyncEnabled = true;
merge(
// do an initial sync immediately
of(true),
// re-sync at regular interval
interval(this.SYNC_INTERVAL),
// and immediately sync to upload any local changes
this.changes(),
)
.pipe(
debounceTime(500),
mergeMap(() => {
if (this.syncState.value == SyncState.STARTED) {
return EMPTY;
} else {
// catch errors so the outer observable stays alive without re-subscribing
// (re-subscribing via retry would leak EventEmitter listeners on PouchDB)
// sync() already logs errors and updates syncState before re-throwing
return from(
this.sync().catch((err) => {
Logging.debug("liveSync: swallowed sync error", err);
}),
);
}
}),
takeWhile(() => this.liveSyncEnabled),
takeUntil(this.destroy$),
)
.subscribe();
}
}
type SyncResult = PouchDB.Replication.SyncResultComplete<any>;
/** The part of the error in a PouchDB "denied" event that is logged. */
type DeniedDoc = { id?: string; name?: string };
/**
* What of the last synced batch can be reported when a sync fails right after
* it: how much was in flight and which record types, but never the document
* ids themselves - those identify records and have no place in remote
* monitoring (see #4174), while as part of a message they would also open a
* separate issue per batch.
*/
function lastSyncedBatchContext(docIds: string[]): Record<string, unknown> {
return {
lastSyncedBatchSize: docIds.length,
lastSyncedEntityTypes: [...new Set(docIds.map((id) => id.split(":")[0]))],
};
}
/**
* Thrown by {@link SyncedPouchDatabase.ensureSynced} when the sync failed.
* The sync has already logged the failure: callers only need to tell the user.
*/
export class SyncFailedError extends NamedError {
constructor(cause: unknown) {
super("SyncFailedError", "Could not sync with the server", { cause });
}
}
/** Thrown internally when a sync is cancelled for making no progress (a stalled connection). */
class SyncStalledError extends ConnectivityError {
constructor() {
super("Sync cancelled after making no progress");
this.name = "SyncStalledError";
}
}