pending.rs 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. //! The write-ahead record awaiting recovery, and how each kind completes.
  2. //!
  3. //! A commit (reserve → finalize) and an account-version transition (append
  4. //! version → append event) are each more than one store write with no shared
  5. //! transaction, so a crash mid-sequence can leave a half-applied state. Before
  6. //! either mutates anything it persists a [`PendingRecord`] via `SagaStore`; on
  7. //! startup [`Ledger::recover`](super::Ledger::recover) loads every surviving
  8. //! record and drives it to a terminal state through [`PendingRecord::complete`].
  9. //!
  10. //! This module owns the whole write-ahead concept behind one seam: what a
  11. //! pending record *is*, how it is (de)serialized, how it is persisted, and how
  12. //! each kind completes. The completion primitives it calls (`finalize_envelope`,
  13. //! `drive_envelope_saga`) stay on [`Ledger`] because the live commit path shares
  14. //! them; this module sequences them for the recovery path.
  15. use std::sync::Arc;
  16. use kuatia_core::{Account, Envelope, ReservationId, envelope_id};
  17. use kuatia_storage::error::StoreError;
  18. use kuatia_storage::events::{LedgerEvent, LedgerEventKind};
  19. use super::{Ledger, now_millis};
  20. use crate::error::LedgerError;
  21. /// Phase of an in-flight commit, persisted with the write-ahead record so
  22. /// recovery knows whether validation has completed.
  23. #[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
  24. pub(super) enum SagaPhase {
  25. /// Saved before reserve. Validation has not necessarily run, so recovery must
  26. /// re-reserve and re-validate before it can commit.
  27. Reserving,
  28. /// Saved at the start of finalize — after validation passed and just before
  29. /// the consumed postings begin being removed from the reserved index (the
  30. /// point of no return). Recovery rolls forward without re-validating.
  31. Finalizing,
  32. }
  33. /// Write-ahead record for an in-flight commit (reserve → finalize). Persisted
  34. /// before the saga mutates anything and removed once it reaches a terminal
  35. /// state.
  36. #[derive(serde::Serialize, serde::Deserialize)]
  37. pub(super) struct PendingSaga {
  38. pub(super) envelope: Envelope,
  39. pub(super) reservation: ReservationId,
  40. pub(super) phase: SagaPhase,
  41. }
  42. /// Write-ahead record for an in-flight account-version transition
  43. /// (freeze/unfreeze/close). The transition appends a new account version and then
  44. /// its lifecycle event; a crash between the two leaves a version bump with no
  45. /// event. Persisting this before either write lets recovery roll the transition
  46. /// forward, re-appending the (idempotent) event.
  47. #[derive(serde::Serialize, serde::Deserialize)]
  48. pub(super) struct PendingTransition {
  49. /// The next account version to append: version already bumped, flag flipped.
  50. pub(super) next: Account,
  51. /// The lifecycle event paired with this version bump. It carries the target
  52. /// version, so re-appending it on recovery dedups to the original.
  53. pub(super) event: LedgerEventKind,
  54. }
  55. /// The two kinds of write-ahead record the [`SagaStore`] holds, tagged so
  56. /// recovery can tell an envelope commit saga from an account transition and
  57. /// complete each through its own path.
  58. #[derive(serde::Serialize, serde::Deserialize)]
  59. pub(super) enum PendingRecord {
  60. /// A two-step envelope commit saga (reserve → finalize).
  61. Envelope(PendingSaga),
  62. /// A single account-version transition (append version + lifecycle event).
  63. Transition(PendingTransition),
  64. }
  65. impl PendingRecord {
  66. /// A commit write-ahead record at the given phase.
  67. pub(super) fn envelope(
  68. envelope: Envelope,
  69. reservation: ReservationId,
  70. phase: SagaPhase,
  71. ) -> Self {
  72. Self::Envelope(PendingSaga {
  73. envelope,
  74. reservation,
  75. phase,
  76. })
  77. }
  78. /// An account-transition write-ahead record.
  79. pub(super) fn transition(next: Account, event: LedgerEventKind) -> Self {
  80. Self::Transition(PendingTransition { next, event })
  81. }
  82. /// Decode a record from its stored bytes. The single decoder for the
  83. /// write-ahead format, shared by `recover` and the keyed phase read.
  84. pub(super) fn decode(blob: &[u8]) -> Result<Self, LedgerError> {
  85. serde_json::from_slice(blob)
  86. .map_err(|e| LedgerError::Store(StoreError::Internal(e.to_string())))
  87. }
  88. /// The commit phase of an envelope record; `None` for a transition record,
  89. /// which has no phase.
  90. pub(super) fn envelope_phase(&self) -> Option<SagaPhase> {
  91. match self {
  92. Self::Envelope(s) => Some(s.phase),
  93. Self::Transition(_) => None,
  94. }
  95. }
  96. /// Persist this record under `saga_id` (upsert on the id).
  97. pub(super) async fn save(&self, ledger: &Ledger, saga_id: i64) -> Result<(), LedgerError> {
  98. let blob = serde_json::to_vec(self)
  99. .map_err(|e| LedgerError::Store(StoreError::Internal(e.to_string())))?;
  100. ledger.store.save_saga(&saga_id, blob).await?;
  101. Ok(())
  102. }
  103. /// Drive this record to a terminal state and clear it when safe. Called by
  104. /// [`Ledger::recover`](super::Ledger::recover) for every surviving record.
  105. ///
  106. /// A transition rolls forward (any completion error propagates, so recovery
  107. /// retries on the next run). An envelope commit branches on its phase, and
  108. /// its drive/finalize failures are absorbed here (the record is kept for a
  109. /// later run) rather than aborting recovery of the remaining records.
  110. pub(super) async fn complete(
  111. self,
  112. ledger: &Arc<Ledger>,
  113. saga_id: i64,
  114. ) -> Result<(), LedgerError> {
  115. match self {
  116. Self::Transition(PendingTransition { next, event }) => {
  117. complete_transition(ledger, saga_id, next, event).await
  118. }
  119. Self::Envelope(PendingSaga {
  120. envelope,
  121. reservation,
  122. phase,
  123. }) => complete_envelope(ledger, saga_id, envelope, reservation, phase).await,
  124. }
  125. }
  126. }
  127. /// Roll a crash-interrupted transition forward and clear its write-ahead record.
  128. ///
  129. /// Idempotent in every crash window: the version append runs only into an empty
  130. /// version slot (`append_account_version` requires `version == current + 1`, so a
  131. /// blind retry after it applied would fail), and the event carries its target
  132. /// version so re-appending it dedups to the original. The empty-slot guard also
  133. /// subsumes the forward path's is_closed check: a close always bumps the version,
  134. /// so a since-closed account sits at `version >= next.version` and is skipped.
  135. async fn complete_transition(
  136. ledger: &Ledger,
  137. saga_id: i64,
  138. next: Account,
  139. event: LedgerEventKind,
  140. ) -> Result<(), LedgerError> {
  141. // The account is guaranteed to exist here (its version was bumped, or is
  142. // about to be), so a read failure is transient or a real invariant breach,
  143. // not "not found": surface it verbatim so recovery retries.
  144. let current = ledger.store.get_account(&next.id).await?;
  145. if current.version < next.version {
  146. ledger.store.append_account_version(next).await?;
  147. }
  148. ledger
  149. .store
  150. .append_event(&LedgerEvent {
  151. seq: 0,
  152. timestamp: now_millis()?,
  153. kind: event,
  154. })
  155. .await?;
  156. ledger.store.delete_saga(&saga_id).await?;
  157. Ok(())
  158. }
  159. /// Complete a crash-interrupted commit and clear its record when safe.
  160. async fn complete_envelope(
  161. ledger: &Arc<Ledger>,
  162. saga_id: i64,
  163. envelope: Envelope,
  164. reservation: ReservationId,
  165. phase: SagaPhase,
  166. ) -> Result<(), LedgerError> {
  167. // The transfer record is durable, but a full commit is more than the transfer
  168. // row: it also includes the committed event, appended *after* store_transfer.
  169. // A crash in that window leaves the record present yet the event missing, so
  170. // repair the whole end-state (idempotent) before clearing the record.
  171. let tid = envelope_id(&envelope);
  172. if ledger.store.get_transfer(&tid).await?.is_some() {
  173. ledger.append_committed_event(tid).await?;
  174. ledger.store.delete_saga(&saga_id).await?;
  175. return Ok(());
  176. }
  177. match phase {
  178. SagaPhase::Finalizing => {
  179. // Validation passed and the postings are ours; roll forward. Keep the
  180. // record if completion fails so a later run retries.
  181. if ledger
  182. .finalize_envelope(&envelope, reservation)
  183. .await
  184. .is_ok()
  185. {
  186. ledger.store.delete_saga(&saga_id).await?;
  187. }
  188. }
  189. SagaPhase::Reserving => {
  190. // Re-run the validating saga. On failure, delete only if it did not
  191. // reach finalize (clean abort); otherwise keep for the next run.
  192. let result = ledger.drive_envelope_saga(envelope, reservation).await;
  193. let safe_to_delete =
  194. result.is_ok() || ledger.saga_phase(saga_id).await? != Some(SagaPhase::Finalizing);
  195. if safe_to_delete {
  196. ledger.store.delete_saga(&saga_id).await?;
  197. }
  198. }
  199. }
  200. Ok(())
  201. }