Crash Recovery: Reconciliation Loops That Prevent Double Orders

async function findOrphanOrders(exchange, localOrderIds: Set<string>) {
  const exchangeOrders = await exchange.fetchOpenOrders();
  return exchangeOrders.filter(o => !localOrderIds.has(o.clientOrderId));
}

async function findGhostOrders(exchange, localOrders: Order[]) {
  const ghostOrders: Order[] = [];
  for (const local of localOrders) {
    const remote = await exchange.fetchOrder(local.clientOrderId);
    if (!remote || remote.status === 'not_found') {
      ghostOrders.push(local);
    }
  }
  return ghostOrders;
}

async function findStaleFills(exchange, localOrders: Order[]) {
  const staleFills: Order[] = [];
  for (const local of localOrders) {
    if (local.status !== 'filled') {
      const remote = await exchange.fetchOrder(local.clientOrderId);
      if (remote?.status === 'filled') {
        staleFills.push({ local, remote });
      }
    }
  }
  return staleFills;
}

async function reconcileOnStartup(
  exchange: Exchange,
  state: TradingState
): Promise<ReconciliationResult> {
  const result: ReconciliationResult = {
    orphansFound: 0,
    ghostsRemoved: 0,
    fillsBackfilled: 0,
    positionCorrected: false,
  };
 
  // Phase 1: Detect orphan orders (exchange has, we don't)
  const orphans = await findOrphanOrders(exchange, state.orderIds);
  for (const orphan of orphans) {
    // Option A: Cancel if strategy no longer valid
    // Option B: Adopt and track
    await state.adoptOrder(orphan);
    result.orphansFound++;
  }
 
  // Phase 2: Remove ghost orders (we have, exchange doesn't)
  const ghosts = await findGhostOrders(exchange, state.openOrders);
  for (const ghost of ghosts) {
    await state.removeOrder(ghost.clientOrderId);
    result.ghostsRemoved++;
  }
 
  // Phase 3: Backfill stale fills
  const staleFills = await findStaleFills(exchange, state.openOrders);
  for (const { local, remote } of staleFills) {
    await state.processFill(local.clientOrderId, remote.filled, remote.price);
    result.fillsBackfilled++;
  }
 
  // Phase 4: Verify position accuracy
  const exchangePosition = await exchange.fetchPosition(state.symbol);
  if (Math.abs(exchangePosition.size - state.position.size) > 0.0001) {
    state.position.size = exchangePosition.size;
    result.positionCorrected = true;
  }
 
  return result;
}

function generateClientOrderId(
  strategy: string,
  symbol: string,
  timestamp: number
): string {
  // Deterministic: same inputs always produce same ID
  // Unique: different orders get different IDs
  return `${strategy}-${symbol}-${timestamp}-${randomSuffix()}`;
}

async function reconcilePosition(
  exchange: Exchange,
  state: TradingState
): Promise<void> {
  const remote = await exchange.fetchPosition(state.symbol);
  const local = state.position;
 
  const driftSize = Math.abs(remote.size - local.size);
  const driftPct = (driftSize / Math.abs(local.size || 1)) * 100;
 
  if (driftPct > 0.1) { // More than 0.1% drift
    console.warn(`Position drift detected: local=${local.size}, remote=${remote.size}`);
    
    // Update local state to match exchange reality
    state.position.size = remote.size;
    state.position.entryPrice = remote.entryPrice;
    
    // Log for audit
    await auditLog({
      event: 'position_reconciled',
      drift: driftSize,
      localBefore: local.size,
      remoteTruth: remote.size,
    });
  }
}

async function startupSequence(config: BotConfig): Promise<void> {
  // 1. Load state
  const state = await loadState(config.stateFile);
  
  // 2. Create exchange connection (but don't trade yet)
  const exchange = createExchange(config, { tradingEnabled: false });
  
  // 3. Reconcile
  const reconciled = await reconcileOnStartup(exchange, state);
  console.log('Reconciliation complete:', reconciled);
  
  // 4. Verify risk is within limits post-reconciliation
  if (!isWithinRiskLimits(state.position, config.limits)) {
    throw new Error('Position exceeds risk limits after reconciliation');
  }
  
  // 5. Enable trading
  exchange.enableTrading();
  
  // 6. Connect live data
  await exchange.connectWebSocket();
  
  console.log('Trading resumed');
}

async function safeReconciliation(
  exchange: Exchange,
  state: TradingState
): Promise<ReconciliationResult | null> {
  try {
    return await reconcileOnStartup(exchange, state);
  } catch (error) {
    if (error.code === 'RATE_LIMITED') {
      // Wait and retry
      await delay(60_000);
      return await reconcileOnStartup(exchange, state);
    }
    
    if (error.code === 'EXCHANGE_DOWN') {
      // Cannot reconcile—refuse to start
      console.error('Cannot reconcile: exchange unreachable');
      process.exit(1);
    }
    
    // Unknown error—refuse to trade
    console.error('Reconciliation failed:', error);
    return null;
  }
}

// Run light reconciliation every 5 minutes
setInterval(async () => {
  const drift = await checkPositionDrift(exchange, state);
  if (drift > config.driftThreshold) {
    await reconcilePosition(exchange, state);
  }
}, 5 * 60 * 1000);