Sweep for missed events every hour

How to do it

Events get lost when your endpoint is down for longer than the provider retries, or when a deploy drops a request. List payments and refunds created or changed in the last 48 hours. Compare each with the order. Apply a missing transition through the same function the webhook uses. Count every fix and alert when the count is above zero, since each fix is a lost event. Watch the job itself with a heartbeat.

Example

const LOOKBACK_SECONDS = 48 * 60 * 60

async function sweepPayments() {
  let fixes = 0
  const since = Math.floor(Date.now() / 1000) - LOOKBACK_SECONDS
  for await (const payment of stripe.paymentIntents.list({ created: { gte: since }, limit: 100 })) {
    const order = await orders.findByPaymentId(payment.id)
    if (order && needsTransition(order, payment)) {
      await applyPaymentState(order, payment, 'sweep')
      fixes++
    }
  }
  for await (const refund of stripe.refunds.list({ created: { gte: since }, limit: 100 })) {
    if (await applyRefundIfMissing(refund, 'sweep')) fixes++
  }
  metrics.gauge('payments.sweep.fixes', fixes)
}

LOOKBACK_SECONDS = 48 * 60 * 60

def sweep_payments() -> None:
    fixes = 0
    since = int(time.time()) - LOOKBACK_SECONDS
    for payment in stripe.PaymentIntent.list(created={"gte": since}, limit=100).auto_paging_iter():
        order = orders.find_by_payment_id(payment.id)
        if order and needs_transition(order, payment):
            apply_payment_state(order, payment, cause="sweep")
            fixes += 1
    for refund in stripe.Refund.list(created={"gte": since}, limit=100).auto_paging_iter():
        if apply_refund_if_missing(refund, cause="sweep"):
            fixes += 1
    metrics.gauge("payments.sweep.fixes", fixes)

How to check

On staging, turn off the webhook endpoint, pay for an order, and turn the endpoint on again. Within an hour the order is paid, and the job reports one fix.