Настройка RTO (Recovery Time Objective) и RPO (Recovery Point Objective)

Настройка RTO и RPO для критических систем

RTO (Recovery Time Objective) — максимально допустимое время простоя после сбоя. RPO (Recovery Point Objective) — максимально допустимая потеря данных в единицах времени. Эти два параметра определяют всю архитектуру резервирования: чем ниже значения, тем дороже инфраструктура.

Зависимость стоимости от RTO/RPO

Цифры условные — зависят от объёма данных и regional ценообразования.

Определение бизнес-требований

# Расчёт стоимости простоя для определения приемлемого RTO
class RtoCalculator:
    def calculate_downtime_cost(
        self,
        hourly_revenue: float,
        customer_churn_per_hour: float,   # % клиентов, уходящих за каждый час простоя
        penalty_per_sla_violation: float, # штраф по SLA договорам
        avg_customer_lifetime_value: float,
        total_customers: int
    ) -> dict:

        costs_per_hour = {
            'lost_revenue': hourly_revenue,
            'customer_churn': (customer_churn_per_hour / 100) * total_customers
                             * avg_customer_lifetime_value,
            'sla_penalties': penalty_per_sla_violation,
            'recovery_labor': 500  # инженерное время
        }

        total_per_hour = sum(costs_per_hour.values())

        return {
            'cost_per_hour': total_per_hour,
            'breakdown': costs_per_hour,
            'recommended_rto': self._recommend_rto(total_per_hour),
            'max_acceptable_downtime_per_year':
                (total_per_hour * 8760 * 0.001)  # 0.1% годового дохода
        }

    def _recommend_rto(self, hourly_cost: float) -> str:
        if hourly_cost > 100000:
            return "< 5 minutes (active-active)"
        elif hourly_cost > 10000:
            return "< 15 minutes (hot standby)"
        elif hourly_cost > 1000:
            return "< 1 hour (warm standby)"
        else:
            return "< 4 hours (cold standby)"

PostgreSQL: конфигурация для RPO = 5 минут

# postgresql.conf — WAL archiving для PITR
wal_level = replica
archive_mode = on
archive_command = 'pgbackrest --stanza=main archive-push %p'

# Частота checkpoint (влияет на RPO при crash без replica)
checkpoint_timeout = 5min          # checkpoint каждые 5 минут
checkpoint_completion_target = 0.9

# Для streaming replication
max_wal_senders = 10
wal_keep_size = 1GB

# pgbackrest.conf — архив WAL в S3 каждые 5 минут
[global]
repo1-type=s3
repo1-s3-bucket=myapp-wal-archive
repo1-s3-region=eu-central-1
repo1-retention-full=4          # хранить 4 full backup
repo1-retention-diff=14         # 14 diff backup

[main]
pg1-path=/var/lib/postgresql/data
pg1-host=db-replica-1
pg1-host-user=postgres

# WAL archiving: автоматически каждые ~5 минут
archive-push-queue-max=1GiB

# Расписание backup
# crontab для postgres user:
# Full backup раз в неделю
0 1 * * 0 pgbackrest --stanza=main backup --type=full

# Differential backup ежедневно
0 2 * * 1-6 pgbackrest --stanza=main backup --type=diff

# WAL архивируется непрерывно через archive_command

Streaming Replication для RTO = 30 минут

# На primary: создать пользователя репликации
psql -U postgres -c "CREATE USER replicator REPLICATION LOGIN PASSWORD 'strongpassword';"

# pg_hba.conf на primary
host replication replicator replica-ip/32 md5

# Настройка replica
pg_basebackup -h primary-ip -U replicator -D /var/lib/postgresql/data \
  --wal-method=stream --checkpoint=fast --progress

# recovery.conf (PostgreSQL < 12) или postgresql.conf (>= 12)
primary_conninfo = 'host=primary-ip user=replicator password=strongpassword'
recovery_target_timeline = 'latest'

Patroni: автоматический failover (RTO < 30 сек)

# /etc/patroni/patroni.yml
scope: postgres-cluster
namespace: /service/
name: pg-node-1

restapi:
  listen: 0.0.0.0:8008
  connect_address: pg-node-1-ip:8008

etcd3:
  hosts: etcd1:2379,etcd2:2379,etcd3:2379

bootstrap:
  dcs:
    ttl: 30               # через 30 сек без heartbeat — failover
    loop_wait: 10
    retry_timeout: 30
    maximum_lag_on_failover: 1048576  # 1MB: не промоутировать сильно отставшую реплику

  postgresql:
    use_pg_rewind: true
    parameters:
      wal_level: replica
      hot_standby: on
      max_wal_senders: 10
      archive_mode: on
      archive_command: 'pgbackrest --stanza=main archive-push %p'

postgresql:
  listen: 0.0.0.0:5432
  connect_address: pg-node-1-ip:5432
  data_dir: /var/lib/postgresql/data
  authentication:
    replication:
      username: replicator
      password: strongpassword
    superuser:
      username: postgres
      password: adminpassword

tags:
  nofailover: false
  noloadbalance: false

HAProxy: маршрутизация с учётом роли

# haproxy.cfg
frontend postgres_write
  bind *:5432
  default_backend postgres_primary

backend postgres_primary
  option httpchk GET /master
  http-check expect status 200
  server pg-node-1 pg-node-1-ip:5432 check port 8008
  server pg-node-2 pg-node-2-ip:5432 check port 8008
  server pg-node-3 pg-node-3-ip:5432 check port 8008

frontend postgres_read
  bind *:5433
  default_backend postgres_replicas

backend postgres_replicas
  balance roundrobin
  option httpchk GET /replica
  http-check expect status 200
  server pg-node-1 pg-node-1-ip:5432 check port 8008
  server pg-node-2 pg-node-2-ip:5432 check port 8008
  server pg-node-3 pg-node-3-ip:5432 check port 8008

Мониторинг RTO/RPO метрик

# Prometheus exporter: экспортировать реальное отставание реплики
import psycopg2
from prometheus_client import Gauge, start_http_server

replication_lag = Gauge('postgresql_replication_lag_seconds',
                        'Replication lag in seconds',
                        ['replica_host'])

recovery_point = Gauge('postgresql_recovery_point_seconds',
                       'Seconds since last replayed WAL')

def collect_replication_metrics():
    with psycopg2.connect(host='primary-ip', user='monitor', password='pass') as conn:
        cur = conn.cursor()
        cur.execute("""
            SELECT client_addr,
                   extract(epoch from (now() - write_lag)) AS write_lag_sec,
                   extract(epoch from (now() - flush_lag)) AS flush_lag_sec,
                   extract(epoch from (now() - replay_lag)) AS replay_lag_sec
            FROM pg_stat_replication
        """)
        for row in cur.fetchall():
            host, write_lag, flush_lag, replay_lag = row
            replication_lag.labels(replica_host=str(host)).set(replay_lag or 0)

# Prometheus alerting для SLA нарушений
- alert: ReplicationLagCritical
  expr: postgresql_replication_lag_seconds > 300
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: "RPO at risk: replica lag {{ $value }}s > 5min RPO target"

Срок выполнения

Настройка Patroni + pgBackRest + HAProxy для достижения RTO < 30 мин и RPO < 5 мин — 3–5 рабочих дней.