src/app/core/database/pouchdb/synced-pouch-database.ts

Description

Thrown internally when a sync is cancelled for making no progress (a stalled connection).

Extends

ConnectivityError

Relationships

Used by

No results matching.

Depends on

Index

Properties

Constructor

constructor()

Properties

Readonly classification
Type : ErrorClassification
Inherited from TypedError

The kind of failure: an own property, so that a copy of the error that lost its class (as PouchDB's createError() makes) still has it.

Readonly Optional status
Type : number
Inherited from TypedError

The HTTP status of the failed request, if any.

Readonly Optional details
Type : Record<string, unknown>
Inherited from NamedError

Context for the report, kept out of the message so that it does not split the issue.

import { PouchDatabase } from "./pouch-database";
import { toTypedDatabaseError } from "./database-exception";
import { Logging } from "../../logging/logging.service";
import { NamedError } from "../../logging/reported-errors";
import { KeycloakAuthService } from "../../session/auth/keycloak/keycloak-auth.service";
import { NgZone } from "@angular/core";
import { RemotePouchDatabase } from "./remote-pouch-database";
import {
  debounceTime,
  filter,
  mergeMap,
  takeUntil,
  takeWhile,
} from "rxjs/operators";
import { SyncState } from "../../session/session-states/sync-state.enum";
import {
  LoginStateSubject,
  SyncStateSubject,
} from "../../session/session-type";
import { EMPTY, from, interval, merge, of } from "rxjs";
import { LoginState } from "../../session/session-states/login-state.enum";
import { NotAvailableOfflineError } from "../../session/not-available-offline.error";
import { AlertService } from "../../alerts/alert.service";
import { QueryOptions } from "../database";
import { PouchdbCorruptionRecoveryService } from "./pouchdb-corruption-recovery.service";
import { isDocumentWriteError } from "./pouchdb-errors";
import { ConnectivityError } from "#src/app/utils/backend-errors";
import { LAST_SYNC_KEY_PREFIX } from "#src/bootstrap-reset";

/**
 * An alternative implementation of PouchDatabase that additionally
 * provides functionality to sync with a remote CouchDB.
 */
export class SyncedPouchDatabase extends PouchDatabase {
  get LAST_SYNC_KEY(): string | undefined {
    if (!this.pouchDB?.name) {
      return undefined;
    }
    return LAST_SYNC_KEY_PREFIX + this.pouchDB.name;
  }

  /**
   * Destroy the local database.
   * Its last completed sync is forgotten, so that a new database with the same name
   * is not taken for local data that was lost (see DatabaseResolverService).
   */
  override async destroy(): Promise<any> {
    const lastSyncKey = this.LAST_SYNC_KEY;
    const result = await super.destroy();
    if (lastSyncKey) {
      localStorage.removeItem(lastSyncKey);
    }
    return result;
  }

  POUCHDB_SYNC_BATCH_SIZE = 100;
  SYNC_INTERVAL = 30000;

  /**
   * Abort a sync that makes no progress within this window (ms).
   * Push writes have no per-request abort timeout, so a stale/half-open
   * connection can leave the replication promise unsettled forever - which would
   * keep syncState at STARTED and block liveSync from starting any further sync.
   *
   * The timer resets on every replication progress event, so this is an
   * inactivity window per batch (see POUCHDB_SYNC_BATCH_SIZE) and not a budget
   * for the total sync duration - a long initial sync of a large database keeps
   * resetting the timer and is never cancelled while it makes progress.
   * The window still has to cover the slowest legitimate gap between events:
   * the checkpoint lookup and first `_changes` response before any batch
   * arrives, and a single batch of documents with attachments over a poor
   * connection. If that gap is exceeded no batch completes, so no checkpoint is
   * written and every retry restarts from the same position - which is why this
   * is set generously rather than close to a normal sync's duration.
   */
  SYNC_STALL_TIMEOUT = 300000;

  private readonly navigator: Navigator;
  private readonly loginStateSubject: LoginStateSubject;
  private readonly alertService?: AlertService;
  private readonly corruptionRecovery?: PouchdbCorruptionRecoveryService;
  private remoteDatabase: RemotePouchDatabase;
  private syncState: SyncStateSubject = new SyncStateSubject();

  /**
   * Get the internal sync state subject for this database (not the global one).
   * Useful for observing when this specific database's sync completes.
   */
  get localSyncState(): SyncStateSubject {
    return this.syncState;
  }

  constructor(
    dbName: string,
    authService: KeycloakAuthService,
    globalSyncState: SyncStateSubject,
    navigator: Navigator,
    loginStateSubject: LoginStateSubject,
    ngZone?: NgZone,
    alertService?: AlertService,
    corruptionRecovery?: PouchdbCorruptionRecoveryService,
  ) {
    super(dbName, globalSyncState, ngZone);
    this.navigator = navigator;
    this.loginStateSubject = loginStateSubject;
    this.alertService = alertService;
    this.corruptionRecovery = corruptionRecovery;

    this.remoteDatabase = new RemotePouchDatabase(
      dbName,
      authService,
      undefined,
      ngZone,
      this.alertService,
    );

    this.logSyncContext();
    this.syncState
      .pipe(
        takeUntil(this.destroy$),
        filter((state) => state === SyncState.COMPLETED),
      )
      .subscribe(() => {
        const lastSyncTime = new Date().toISOString();
        localStorage.setItem(this.LAST_SYNC_KEY, lastSyncTime);
        this.logSyncContext();
      });

    // forward sync state to global sync state (combining state from all synced databases)
    this.syncState
      .pipe(takeUntil(this.destroy$))
      .subscribe((state: SyncState) => this.globalSyncState.next(state));

    // Start live sync whenever the user is logged in.
    // Note: if the user logged in offline (no Keycloak token), syncing will
    // hit a 401 and the RemotePouchDatabase fetch interceptor will trigger a
    // Keycloak login redirect. This is intentional — we want users to
    // authenticate online when connectivity is available.
    this.loginStateSubject
      .pipe(
        takeUntil(this.destroy$),
        filter((state) => state === LoginState.LOGGED_IN),
      )
      .subscribe(() => this.liveSync());
  }

  /**
   * Initializes the PouchDB with local indexeddb as well as a remote server connection for syncing.
   * @param dbName local database name (for the current user)
   * @param remoteDbName (optional) remote database name (if different from local browser database name)
   */
  override init(dbName?: string, remoteDbName?: string) {
    super.init(dbName ?? this.dbName, undefined, true);

    // keep remote database on default name (e.g. "app" instead of "user_uuid-app")
    this.remoteDatabase.init(remoteDbName, { trackLostPermissions: true });
  }

  /** doc counts of the most recent completed sync (for logging context) */
  private lastSyncStats?: { pushed?: number; pulled?: number };

  private async logSyncContext() {
    const lastSyncTime = localStorage.getItem(this.LAST_SYNC_KEY);
    // the context key includes the database name: every synced database (app,
    // notifications, ...) writes its own block. With a shared key the last
    // writer would overwrite the others, so a reported event could show the
    // sync state of a different database than the one it is about.
    Logging.addContext(`Aam Digital sync: ${this.dbName}`, {
      db: this.dbName,
      "last sync completed": lastSyncTime,
      "last sync docs pushed": this.lastSyncStats?.pushed,
      "last sync docs pulled": this.lastSyncStats?.pulled,
    });
  }

  /**
   * Execute a (one-time) sync between the local and server database.
   */
  async sync(
    options: PouchDB.Replication.SyncOptions = {},
  ): Promise<SyncResult> {
    if (!this.navigator.onLine) {
      Logging.debug("Not syncing because offline");
      this.syncState.next(SyncState.UNSYNCED);
      return {};
    }

    let localDb: PouchDB.Database;
    let isFirstSync: boolean;
    try {
      localDb = await this.getPouchDBOnceReady();
      isFirstSync = (await localDb.info()).doc_count === 0;
    } catch (err) {
      // e.g. a broken IndexedDB: logged like any failed sync (see ensureSynced)
      const failure = toTypedDatabaseError(err);
      await this.handleSyncFailure(failure, []);
      throw failure;
    }
    if (isFirstSync) {
      // On first sync there are no local docs, so skip lost-permission tracking & purge
      this.remoteDatabase.trackLostPermissions = false;
    }

    this.syncState.next(SyncState.STARTED);

    // Track the last batch of synced doc IDs for diagnostics on write failures
    let lastSyncedDocIds: string[] = [];

    // Run PouchDB sync/replication outside Angular zone to:
    //  - avoid wasted change-detection cycles for internal sync chatter
    //  - prevent expected internal rejections (e.g. transient 404s for
    //    not-yet-existing remote DBs) from being routed to Angular's
    //    ErrorHandler / Sentry. Outer .then/.catch below still handle them
    //    explicitly and re-enter the zone for state updates.
    const createSyncHandler = () =>
      localDb.sync(this.remoteDatabase.getPouchDB(), {
        batch_size: this.POUCHDB_SYNC_BATCH_SIZE,
        ...options,
      });
    const syncHandler = this.ngZone
      ? this.ngZone.runOutsideAngular(createSyncHandler)
      : createSyncHandler();

    // Guard against a stalled sync (see SYNC_STALL_TIMEOUT): if replication makes
    // no progress within the timeout, cancel it so the promise settles and
    // liveSync can retry, instead of staying blocked in STARTED forever.
    let stallTimer: ReturnType<typeof setTimeout>;
    let rejectStalled: (err: any) => void;
    const stallGuard = new Promise<never>((_, reject) => {
      rejectStalled = reject;
    });
    const armStallTimer = () => {
      clearTimeout(stallTimer);
      stallTimer = setTimeout(() => {
        Logging.debug(`sync stalled, cancelling to allow retry`, {
          db: this.dbName,
        });
        // Reject BEFORE cancelling: PouchDB's replication thenable resolves
        // (fires "complete") on cancel(), so cancelling first could let
        // Promise.race take the success path and mark a stalled sync COMPLETED.
        rejectStalled(new SyncStalledError());
        syncHandler.cancel();
      }, this.SYNC_STALL_TIMEOUT);
    };

    syncHandler.on("change", (info) => {
      armStallTimer();
      lastSyncedDocIds =
        info?.change?.docs?.map((d) => d._id).filter(Boolean) ?? [];
    });
    syncHandler.on("active", () => armStallTimer());
    syncHandler.on("paused", () => armStallTimer());
    // per-doc rejections (e.g. 401/403 during push) do not fail the overall
    // sync, but leave the rejected change in the local database only
    const deniedDocs: DeniedDoc[] = [];
    syncHandler.on("denied", (event: { doc?: DeniedDoc }) =>
      deniedDocs.push(event?.doc ?? {}),
    );
    armStallTimer();

    return Promise.race([syncHandler, stallGuard])
      .then(async (res) => {
        if (res) res["dbName"] = this.dbName; // add for debugging information
        Logging.debug("sync completed", res);
        this.lastSyncStats = {
          pushed: (res as SyncResult)?.push?.docs_written,
          pulled: (res as SyncResult)?.pull?.docs_written,
        };
        if (!isFirstSync) {
          await this.purgeDocsWithLostPermissions();
        }
        this.syncState.next(SyncState.COMPLETED);
        return res as SyncResult;
      })
      .catch(async (err) => {
        // Handle 404 errors for notifications database (may not exist yet if no event was triggered)
        if (this.isNotificationsDatabase() && err?.status === 404) {
          Logging.debug(
            "Notifications database does not exist yet on server - this may be expected",
            err,
          );
          this.syncState.next(SyncState.COMPLETED);
          return {};
        }

        // the replication fails with a copy of the error, which loses the class of a typed error
        const failure = toTypedDatabaseError(err);
        await this.handleSyncFailure(failure, lastSyncedDocIds);
        throw failure;
      })
      .finally(() => {
        clearTimeout(stallTimer);
        if (isFirstSync) {
          this.remoteDatabase.trackLostPermissions = true;
        }
        this.logDeniedDocs(deniedDocs);
      });
  }

  /**
   * Report the docs the server refused during one sync in a single warning,
   * since a single push can be refused for hundreds of docs at once.
   * Only counts and entity types, no document ids (see #4174).
   */
  private logDeniedDocs(deniedDocs: DeniedDoc[]) {
    if (deniedDocs.length === 0) {
      return;
    }
    const ids = deniedDocs.map((doc) => doc.id).filter(Boolean);
    Logging.warn("sync: server denied replication of documents", {
      db: this.dbName,
      denied: deniedDocs.length,
      errors: [...new Set(deniedDocs.map((doc) => doc.name))],
      entityTypes: [...new Set(ids.map((id) => id.split(":")[0]))],
    });
  }

  /**
   * Log a failed sync according to its cause (only once: callers like
   * {@link ensureSynced} rely on this) and mark the sync state as failed.
   */
  private async handleSyncFailure(err: any, lastSyncedDocIds: string[]) {
    await this.logSyncFailure(err, lastSyncedDocIds);
    this.syncState.next(SyncState.FAILED);
  }

  private async logSyncFailure(err: any, lastSyncedDocIds: string[]) {
    const context = {
      db: this.dbName,
      ...lastSyncedBatchContext(lastSyncedDocIds),
    };
    if (isDocumentWriteError(err)) {
      Logging.warn(
        `sync failed: document write error (possible oversized document)`,
        context,
        err,
      );
      return;
    }

    // reported (with the offer to reset the local data) by the recovery service
    const reportedAsCorruption =
      await this.corruptionRecovery?.handleKnownMultiTabCorruption(
        err,
        "sync failed: likely multi-tab IndexedDB corruption",
        context,
        this.pouchDB,
      );
    if (reportedAsCorruption) {
      return;
    }

    // at debug if offline or the session expired (see `Logging`): retried later
    Logging.warn(`sync failed`, { db: this.dbName }, err);
  }

  override async put(object: any, forceOverwrite = false): Promise<any> {
    try {
      return await super.put(object, forceOverwrite);
    } catch (err) {
      await this.corruptionRecovery?.handleKnownMultiTabCorruption(
        err,
        "put failed: likely multi-tab IndexedDB corruption",
        { db: this.dbName },
        this.pouchDB,
      );
      throw err;
    }
  }

  override query(
    fun: string | ((doc: any, emit: any) => void),
    options: QueryOptions,
  ): Promise<any> {
    return super.query(fun, options).catch(async (err) => {
      await this.corruptionRecovery?.handleKnownMultiTabCorruption(
        err,
        "query failed: likely multi-tab IndexedDB corruption",
        { db: this.dbName },
        this.pouchDB,
      );
      throw err;
    });
  }

  /**
   * How many document ids to include as an example in the logs below, which can
   * report on thousands of documents (the count is logged in full alongside).
   */
  private readonly SAMPLE_IDS_LOGGED = 5;

  /**
   * Purge local documents for which the server reported lost permissions
   * during the most recent sync's `_changes` calls.
   *
   * The server reports every document changed since this client's checkpoint
   * that the user may not read - it has no knowledge of which of those the
   * client actually holds locally. Most reported ids therefore do not exist
   * locally at all and purging them is a no-op. Only an actual local deletion
   * is logged with details, so that the log stays a signal for data being
   * removed rather than a count of what the server reported. That is the
   * expected outcome of a change of permissions, so it is not reported to
   * remote monitoring; only a failed purge is.
   */
  private async purgeDocsWithLostPermissions(): Promise<void> {
    const reportedIds = this.remoteDatabase
      .collectAndClearLostPermissions()
      // design docs for indices are managed locally (and shouldn't be synced anyway)
      .filter((id) => !id.startsWith("_design/"));

    if (reportedIds.length === 0) {
      return;
    }

    if (this.adapter === "idb") {
      // a known limitation, not an event: the legacy "idb" adapter does not implement purge()
      // (LocalPermissionEnforcerService destroys the local db instead where necessary)
      Logging.debug(
        "sync: cannot purge local docs on the legacy idb adapter, skipping",
        { db: this.dbName, reportedByServer: reportedIds.length },
      );
      return;
    }

    const purgedIds: string[] = [];
    const failedIds: string[] = [];
    let notPresentLocally = 0;
    let firstError: unknown;

    for (const _id of reportedIds) {
      try {
        if (await this.purge(_id)) {
          purgedIds.push(_id);
        } else {
          notPresentLocally++;
        }
      } catch (err) {
        failedIds.push(_id);
        firstError ??= err;
      }
    }

    const stats = {
      db: this.dbName,
      purged: purgedIds.length,
      notPresentLocally,
      failed: failedIds.length,
      reportedByServer: reportedIds.length,
    };

    if (purgedIds.length > 0) {
      // expected after a change of permissions: the user can no longer read these docs,
      // so local changes to them could not have been synced anyway
      Logging.debug(
        "sync: purged local docs that the server reported as no longer permitted",
        {
          ...stats,
          sampleIds: purgedIds.slice(0, this.SAMPLE_IDS_LOGGED),
          // the sync triggering this purge has replicated but not yet recorded its
          // own timestamp, so this is the previously completed sync - i.e. how long
          // this device had been out of sync *before* the sync that just succeeded.
          previousSyncCompleted: localStorage.getItem(this.LAST_SYNC_KEY),
        },
      );
    } else {
      Logging.debug(
        "sync: no local docs to purge for server-reported lost permissions",
        stats,
      );
    }

    if (failedIds.length > 0) {
      // the local database keeps docs the user is no longer allowed to read
      Logging.warn(
        "sync: failed to purge local docs that the server reported as no longer permitted",
        { ...stats, sampleIds: failedIds.slice(0, this.SAMPLE_IDS_LOGGED) },
        firstError,
      );
    }
  }

  /**
   * Force a full re-check against the remote DB without deleting local data.
   * Uses `checkpoint: false` once so PouchDB ignores previous checkpoints for this run.
   */
  async resetSync(): Promise<void> {
    Logging.debug(`triggering full re-sync for "${this.dbName}"`);
    await this.sync({ checkpoint: false });
  }

  /**
   * Ensure the database is synced with the remote server.
   * Throws {@link NotAvailableOfflineError} if offline, or a {@link SyncFailedError}
   * if the sync failed (which `sync()` has already logged).
   * Otherwise triggers a one-time sync and resolves when complete.
   */
  async ensureSynced(): Promise<void> {
    if (!this.navigator.onLine) {
      throw new NotAvailableOfflineError(
        "Failed to ensure synced. Cannot sync database while offline.",
      );
    }

    try {
      await this.sync();
    } catch (err) {
      throw new SyncFailedError(err);
    }

    if (this.syncState.value === SyncState.UNSYNCED) {
      throw new NotAvailableOfflineError(
        "Failed to ensure synced. SyncState still reported as UNSYNCED.",
      );
    }
  }

  /**
   * Continuous syncing in background.
   */
  liveSyncEnabled: boolean;

  liveSync() {
    this.liveSyncEnabled = true;

    merge(
      // do an initial sync immediately
      of(true),
      // re-sync at regular interval
      interval(this.SYNC_INTERVAL),
      // and immediately sync to upload any local changes
      this.changes(),
    )
      .pipe(
        debounceTime(500),
        mergeMap(() => {
          if (this.syncState.value == SyncState.STARTED) {
            return EMPTY;
          } else {
            // catch errors so the outer observable stays alive without re-subscribing
            // (re-subscribing via retry would leak EventEmitter listeners on PouchDB)
            // sync() already logs errors and updates syncState before re-throwing
            return from(
              this.sync().catch((err) => {
                Logging.debug("liveSync: swallowed sync error", err);
              }),
            );
          }
        }),
        takeWhile(() => this.liveSyncEnabled),
        takeUntil(this.destroy$),
      )
      .subscribe();
  }
}

type SyncResult = PouchDB.Replication.SyncResultComplete<any>;

/** The part of the error in a PouchDB "denied" event that is logged. */
type DeniedDoc = { id?: string; name?: string };

/**
 * What of the last synced batch can be reported when a sync fails right after
 * it: how much was in flight and which record types, but never the document
 * ids themselves - those identify records and have no place in remote
 * monitoring (see #4174), while as part of a message they would also open a
 * separate issue per batch.
 */
function lastSyncedBatchContext(docIds: string[]): Record<string, unknown> {
  return {
    lastSyncedBatchSize: docIds.length,
    lastSyncedEntityTypes: [...new Set(docIds.map((id) => id.split(":")[0]))],
  };
}

/**
 * Thrown by {@link SyncedPouchDatabase.ensureSynced} when the sync failed.
 * The sync has already logged the failure: callers only need to tell the user.
 */
export class SyncFailedError extends NamedError {
  constructor(cause: unknown) {
    super("SyncFailedError", "Could not sync with the server", { cause });
  }
}

/** Thrown internally when a sync is cancelled for making no progress (a stalled connection). */
class SyncStalledError extends ConnectivityError {
  constructor() {
    super("Sync cancelled after making no progress");
    this.name = "SyncStalledError";
  }
}

results matching ""

    No results matching ""