Enterprise Kubernetes for a critical payments platform on AWS

provider "aws" {
  region = var.region            # ca-central-1 or ca-west-1
  default_tags {
    tags = {
      Platform           = "payments"
      Environment        = var.environment    # prod | perf | test | dev
      Region             = var.region
      DataClassification = var.data_class      # restricted | confidential | internal
      Residency          = "canada-only"
      Owner              = "payments-platform"
      ManagedBy          = "terraform"
      CostCenter         = var.cost_center
    }
  }
}

# Naming convention: pay-{env}-{region-short}-{component}
# Example: pay-prod-cac1-ledger,  pay-prod-caw1-eks
locals {
  region_short = { "ca-central-1" = "cac1", "ca-west-1" = "caw1" }
  name_prefix  = "pay-${var.environment}-${local.region_short[var.region]}"
}

{
  "Version": "2012-10-17",
  "Statement": [{
    "Sid": "DenyOutsideCanada",
    "Effect": "Deny",
    "NotAction": [
      "iam:*", "organizations:*", "route53:*", "cloudfront:*",
      "waf:*", "wafv2:*", "support:*", "sts:*", "budgets:*"
    ],
    "Resource": "*",
    "Condition": {
      "StringNotEquals": {
        "aws:RequestedRegion": ["ca-central-1", "ca-west-1"]
      }
    }
  }]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "DenyRootUser",
      "Effect": "Deny",
      "Action": "*",
      "Resource": "*",
      "Condition": { "StringLike": { "aws:PrincipalArn": "arn:aws:iam::*:root" } }
    },
    {
      "Sid": "ProtectSecurityControls",
      "Effect": "Deny",
      "Action": [
        "guardduty:DeleteDetector", "guardduty:DisassociateFromMasterAccount",
        "cloudtrail:StopLogging", "cloudtrail:DeleteTrail",
        "config:DeleteConfigurationRecorder", "config:StopConfigurationRecorder",
        "securityhub:DisableSecurityHub"
      ],
      "Resource": "*"
    },
    {
      "Sid": "NoPublicS3",
      "Effect": "Deny",
      "Action": "s3:PutBucketPublicAccessBlock",
      "Resource": "*",
      "Condition": { "Bool": { "s3:PublicAccessBlockConfiguration": "false" } }
    },
    {
      "Sid": "RequireImdsV2",
      "Effect": "Deny",
      "Action": "ec2:RunInstances",
      "Resource": "arn:aws:ec2:*:*:instance/*",
      "Condition": { "StringNotEquals": { "ec2:MetadataHttpTokens": "required" } }
    }
  ]
}

resource "aws_guardduty_organization_admin_account" "this" {
  admin_account_id = var.security_tooling_account_id
}

resource "aws_securityhub_organization_admin_account" "this" {
  admin_account_id = var.security_tooling_account_id
}

# Per-account baseline (applied by stack set to every workload account)
resource "aws_guardduty_detector" "this" {
  enable = true
  datasources {
    kubernetes { audit_logs { enable = true } }
    malware_protection { scan_ec2_instance_with_findings { ebs_volumes { enable = true } } }
  }
}

resource "aws_inspector2_enabler" "this" {
  account_ids    = [data.aws_caller_identity.current.account_id]
  resource_types = ["ECR", "EC2", "LAMBDA"]
}

data "aws_networkmanager_core_network_policy_document" "this" {
  core_network_configuration {
    asn_ranges = ["64512-64555"]
    edge_locations { location = "ca-central-1" }
    edge_locations { location = "ca-west-1" }
  }

  segments { name = "production"    isolate_attachments = false require_attachment_acceptance = true }
  segments { name = "nonproduction" isolate_attachments = false require_attachment_acceptance = true }

  segment_actions {
    action  = "share"
    segment = "production"
    share_with = ["production"]   # production talks only to production
  }

  attachment_policies {
    rule_number     = 100
    condition_logic = "and"
    conditions { type = "tag-value" key = "Environment" operator = "equals" value = "prod" }
    action { association_method = "constant" segment = "production" }
  }
  attachment_policies {
    rule_number     = 200
    condition_logic = "or"
    conditions { type = "tag-value" key = "Environment" operator = "equals" value = "dev" }
    conditions { type = "tag-value" key = "Environment" operator = "equals" value = "test" }
    action { association_method = "constant" segment = "nonproduction" }
  }
}

resource "aws_dx_gateway" "core" {
  name            = "pay-dxgw"
  amazon_side_asn = 64600
}

# One transit virtual interface per dedicated connection (two for redundancy)
resource "aws_dx_transit_virtual_interface" "vif" {
  for_each       = toset(["conn-a", "conn-b"])
  connection_id  = var.dx_connection_ids[each.key]
  dx_gateway_id  = aws_dx_gateway.core.id
  name           = "pay-tvif-${each.key}"
  vlan           = var.dx_vlans[each.key]
  address_family = "ipv4"
  bgp_asn        = 65000
  mtu            = 8500            # jumbo frames for replication throughput
}

# Associate the Direct Connect gateway with the Cloud WAN core network
resource "aws_networkmanager_dx_gateway_attachment" "this" {
  core_network_id            = var.core_network_id
  direct_connect_gateway_arn = aws_dx_gateway.core.arn
  edge_locations             = ["ca-central-1", "ca-west-1"]
}

resource "aws_route53_health_check" "region" {
  for_each          = toset(["cac1", "caw1"])
  fqdn              = "ingress-${each.key}.pay.internal"
  type              = "HTTPS"
  resource_path     = "/healthz"
  port              = 443
  failure_threshold = 2
  request_interval  = 10           # fast detection, ~20 s to unhealthy
}

resource "aws_route53_record" "api" {
  for_each       = { cac1 = "ca-central-1", caw1 = "ca-west-1" }
  zone_id        = aws_route53_zone.private.zone_id
  name           = "api.pay.internal"
  type           = "A"
  set_identifier = each.key
  health_check_id = aws_route53_health_check.region[each.key].id
  latency_routing_policy { region = each.value }
  alias {
    name                   = var.nlb_dns[each.key]
    zone_id                = var.nlb_zone[each.key]
    evaluate_target_health = true
  }
}

locals {
  interface_endpoints = [
    "ecr.api", "ecr.dkr", "sts", "secretsmanager", "kms",
    "logs", "monitoring", "ssm", "ssmmessages", "ec2messages",
    "eks", "elasticloadbalancing", "kafka", "sts"
  ]
}

resource "aws_vpc_endpoint" "iface" {
  for_each            = toset(local.interface_endpoints)
  vpc_id             = var.vpc_id
  service_name       = "com.amazonaws.${var.region}.${each.key}"
  vpc_endpoint_type  = "Interface"
  subnet_ids         = var.endpoint_subnet_ids
  security_group_ids = [aws_security_group.endpoints.id]
  private_dns_enabled = true
}

# S3 reached through a gateway endpoint (no hourly cost, route-table based)
resource "aws_vpc_endpoint" "s3" {
  vpc_id            = var.vpc_id
  service_name      = "com.amazonaws.${var.region}.s3"
  vpc_endpoint_type = "Gateway"
  route_table_ids   = var.private_route_table_ids
}

resource "aws_networkfirewall_rule_group" "egress_allow" {
  capacity = 256
  name     = "pay-egress-allow"
  type     = "STATEFUL"
  rule_group {
    rule_variables {
      ip_sets {
        key = "HOME_NET"
        ip_set { definition = ["10.20.0.0/16"] }
      }
    }
    rules_source {
      rules_source_list {
        generated_rules_type = "ALLOWLIST"
        target_types         = ["TLS_SNI", "HTTP_HOST"]
        targets = [
          ".ca-central-1.amazonaws.com",
          ".ca-west-1.amazonaws.com",
          ".pay-partner.example"          # named partner ranges only
        ]
      }
    }
    stateful_rule_options { rule_order = "STRICT_ORDER" }
  }
}

resource "aws_networkfirewall_firewall_policy" "this" {
  name = "pay-egress-policy"
  firewall_policy {
    stateless_default_actions          = ["aws:forward_to_sfe"]
    stateless_fragment_default_actions = ["aws:forward_to_sfe"]
    stateful_default_actions           = ["aws:drop_established", "aws:alert_established"]
    stateful_engine_options { rule_order = "STRICT_ORDER" }
    stateful_rule_group_reference { resource_arn = aws_networkfirewall_rule_group.egress_allow.arn priority = 100 }
  }
}

resource "aws_security_group" "ledger" {
  name   = "${local.name_prefix}-ledger"
  vpc_id = var.vpc_id
}
resource "aws_vpc_security_group_ingress_rule" "ledger_from_nodes" {
  security_group_id            = aws_security_group.ledger.id
  referenced_security_group_id = aws_security_group.nodes.id
  from_port = 5432  to_port = 6432  ip_protocol = "tcp"
}
# Cross-region replication peer reached over Cloud WAN, restricted to the partner ledger CIDR
resource "aws_vpc_security_group_ingress_rule" "ledger_replication" {
  security_group_id = aws_security_group.ledger.id
  cidr_ipv4         = var.partner_region_data_cidr   # e.g. 10.21.4.0/22
  from_port = 5432  to_port = 5432  ip_protocol = "tcp"
}

resource "aws_eks_cluster" "this" {
  name     = "${local.name_prefix}-eks"
  role_arn = aws_iam_role.cluster.arn
  version  = "1.33"

  vpc_config {
    subnet_ids              = var.app_subnet_ids
    endpoint_private_access = true
    endpoint_public_access  = false                  # private only
    security_group_ids      = [aws_security_group.cluster.id]
  }

  access_config {
    authentication_mode                         = "API"
    bootstrap_cluster_creator_admin_permissions = false
  }

  encryption_config {
    provider { key_arn = aws_kms_key.eks_secrets.arn }
    resources = ["secrets"]                          # envelope-encrypt etcd secrets
  }

  kubernetes_network_config {
    ip_family         = "ipv4"
    service_ipv4_cidr = "172.20.0.0/16"
  }

  enabled_cluster_log_types = [
    "api", "audit", "authenticator", "controllerManager", "scheduler"
  ]
}

# Access by role, least privilege, no shared admin
resource "aws_eks_access_entry" "platform" {
  cluster_name  = aws_eks_cluster.this.name
  principal_arn = aws_iam_role.platform_admin.arn
  type          = "STANDARD"
}
resource "aws_eks_access_policy_association" "platform" {
  cluster_name  = aws_eks_cluster.this.name
  principal_arn = aws_iam_role.platform_admin.arn
  policy_arn    = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
  access_scope { type = "cluster" }
}

resource "aws_eks_addon" "vpc_cni" {
  cluster_name  = aws_eks_cluster.this.name
  addon_name    = "vpc-cni"
  addon_version = "v1.19.2-eksbuild.1"
  pod_identity_association {
    role_arn        = aws_iam_role.vpc_cni.arn
    service_account = "aws-node"
  }
  configuration_values = jsonencode({
    env = {
      ENABLE_PREFIX_DELEGATION = "true"
      WARM_PREFIX_TARGET       = "1"
      ENABLE_POD_ENI           = "true"     # security groups for pods
    }
  })
}

resource "aws_eks_addon" "pod_identity" {
  cluster_name  = aws_eks_cluster.this.name
  addon_name    = "eks-pod-identity-agent"
  addon_version = "v1.3.4-eksbuild.1"
}

# coredns, kube-proxy, aws-ebs-csi-driver follow the same pattern, version pinned.

resource "aws_eks_node_group" "system" {
  cluster_name    = aws_eks_cluster.this.name
  node_group_name = "system"
  node_role_arn   = aws_iam_role.node.arn
  subnet_ids      = var.app_subnet_ids
  ami_type        = "BOTTLEROCKET_ARM_64"
  capacity_type   = "ON_DEMAND"
  instance_types  = ["m7g.large"]
  scaling_config { min_size = 3  max_size = 6  desired_size = 3 }
  update_config   { max_unavailable_percentage = 33 }
  labels = { "pool" = "system" }
  taint { key = "CriticalAddonsOnly" value = "true" effect = "NO_SCHEDULE" }
  launch_template { id = aws_launch_template.system.id  version = "$Latest" }
}
# Launch template enforces IMDSv2 and encrypted volumes
resource "aws_launch_template" "system" {
  name = "${local.name_prefix}-system"
  metadata_options { http_tokens = "required"  http_put_response_hop_limit = 1 }
  block_device_mappings {
    device_name = "/dev/xvda"
    ebs { volume_size = 40  volume_type = "gp3"  encrypted = true  kms_key_id = aws_kms_key.ebs.arn }
  }
}

apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: bottlerocket-arm
spec:
  amiFamily: Bottlerocket
  amiSelectorTerms:
    - alias: bottlerocket@latest
  role: KarpenterNodeRole-pay
  subnetSelectorTerms:
    - tags: { "kubernetes.io/role/internal-elb": "1", "Tier": "application" }
  securityGroupSelectorTerms:
    - tags: { "karpenter.sh/discovery": "pay-prod-cac1-eks" }
  metadataOptions:
    httpTokens: required
    httpPutResponseHopLimit: 1
  blockDeviceMappings:
    - deviceName: /dev/xvdb
      ebs: { volumeSize: 80Gi, volumeType: gp3, encrypted: true, kmsKeyID: "alias/pay-ebs" }
  tags: { "karpenter.sh/discovery": "pay-prod-cac1-eks", "Tier": "application" }

apiVersion: karpenter.sh/v1
kind: NodePool
metadata: { name: tier0-app }
spec:
  template:
    metadata:
      labels: { pool: tier0, workload-tier: "0" }
    spec:
      nodeClassRef: { group: karpenter.k8s.aws, kind: EC2NodeClass, name: bottlerocket-arm }
      taints:
        - key: workload-tier
          value: "0"
          effect: NoSchedule
      requirements:
        - { key: kubernetes.io/arch, operator: In, values: ["arm64"] }
        - { key: karpenter.sh/capacity-type, operator: In, values: ["on-demand"] }
        - { key: karpenter.k8s.aws/instance-family, operator: In, values: ["m7g","r7g"] }
        - { key: karpenter.k8s.aws/instance-size, operator: In, values: ["xlarge","2xlarge"] }
  limits: { cpu: "400", memory: 1600Gi }
  disruption:
    consolidationPolicy: WhenEmpty          # tier-0 is conservative, no live consolidation
    consolidateAfter: 5m
    budgets: [ { nodes: "10%" } ]
---
apiVersion: karpenter.sh/v1
kind: NodePool
metadata: { name: tier1-app }
spec:
  template:
    metadata:
      labels: { pool: tier1, workload-tier: "1" }
    spec:
      nodeClassRef: { group: karpenter.k8s.aws, kind: EC2NodeClass, name: bottlerocket-arm }
      requirements:
        - { key: kubernetes.io/arch, operator: In, values: ["arm64"] }
        - { key: karpenter.sh/capacity-type, operator: In, values: ["spot","on-demand"] }
        - { key: karpenter.k8s.aws/instance-family, operator: In, values: ["m7g","c7g","r7g"] }
  limits: { cpu: "1000", memory: 4000Gi }
  disruption:
    consolidationPolicy: WhenEmptyOrUnderutilized   # tier-1 packs aggressively
    consolidateAfter: 30s

resource "aws_eks_fargate_profile" "batch" {
  cluster_name           = aws_eks_cluster.this.name
  fargate_profile_name   = "batch"
  pod_execution_role_arn = aws_iam_role.fargate.arn
  subnet_ids             = var.app_subnet_ids
  selector { namespace = "batch" }
}

apiVersion: v1
kind: Namespace
metadata:
  name: pay-auth
  labels:
    istio.io/dataplane-mode: ambient
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: latest
    workload-tier: "0"
---
apiVersion: v1
kind: ResourceQuota
metadata: { name: quota, namespace: pay-auth }
spec:
  hard:
    requests.cpu: "200"
    requests.memory: 400Gi
    limits.cpu: "400"
    limits.memory: 800Gi
    pods: "500"
---
apiVersion: v1
kind: LimitRange
metadata: { name: limits, namespace: pay-auth }
spec:
  limits:
    - type: Container
      default:        { cpu: "500m", memory: 512Mi }
      defaultRequest: { cpu: "250m", memory: 256Mi }
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: { name: default-deny, namespace: pay-auth }
spec:
  podSelector: {}
  policyTypes: [Ingress, Egress]    # nothing is allowed until a policy opens it

apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: { name: default, namespace: istio-system }
spec:
  mtls: { mode: STRICT }
---
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: { name: allow-ingress-to-auth, namespace: pay-auth }
spec:
  selector: { matchLabels: { app: payment-auth } }
  action: ALLOW
  rules:
    - from:
        - source:
            principals: ["cluster.local/ns/ingress/sa/gateway"]
      to:
        - operation:
            methods: ["POST"]
            paths: ["/v1/authorize"]

apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: pay-gateway
  namespace: ingress
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: external
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internal
spec:
  gatewayClassName: istio
  listeners:
    - name: https
      protocol: HTTPS
      port: 443
      tls:
        mode: Terminate
        certificateRefs: [ { name: pay-tls } ]
      allowedRoutes: { namespaces: { from: Selector, selector: { matchLabels: { ingress: "true" } } } }
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: { name: auth, namespace: pay-auth }
spec:
  parentRefs: [ { name: pay-gateway, namespace: ingress } ]
  hostnames: ["api.pay.internal"]
  rules:
    - matches: [ { path: { type: PathPrefix, value: /v1/authorize } } ]
      backendRefs: [ { name: payment-auth, port: 8080 } ]

apiVersion: external-secrets.io/v1
kind: ClusterSecretStore
metadata: { name: aws-secrets }
spec:
  provider:
    aws:
      service: SecretsManager
      region: ca-central-1
      auth: { jwt: { serviceAccountRef: { name: external-secrets, namespace: external-secrets } } }
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata: { name: ledger-credentials, namespace: pay-auth }
spec:
  refreshInterval: 1h
  secretStoreRef: { name: aws-secrets, kind: ClusterSecretStore }
  target: { name: ledger-credentials, creationPolicy: Owner }
  data:
    - secretKey: dsn
      remoteRef: { key: pay/prod/ledger/dsn }

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata: { name: verify-signed-images }
spec:
  validationFailureAction: Enforce
  webhookTimeoutSeconds: 30
  rules:
    - name: verify-cosign-kms
      match:
        any: [ { resources: { kinds: ["Pod"] } } ]
      verifyImages:
        - imageReferences: ["ACCOUNT.dkr.ecr.ca-central-1.amazonaws.com/pay/*"]
          mutateDigest: true            # pin tag to the verified digest
          required: true
          attestors:
            - count: 1
              entries:
                - keys:
                    kms: "awskms:///alias/pay-cosign"
    - name: disallow-latest
      match: { any: [ { resources: { kinds: ["Pod"] } } ] }
      validate:
        message: "Floating tags are not allowed, use a digest."
        pattern: { spec: { containers: [ { image: "!*:latest" } ] } }

wal_level                  = replica
synchronous_commit         = remote_apply
synchronous_standby_names  = 'ANY 1 (pay_caw1_standby)'   # partner region required
max_wal_senders            = 16
max_replication_slots      = 16
hot_standby                = on
wal_compression            = on
wal_keep_size              = 8GB
max_connections            = 600          # PgBouncer fronts client connections
shared_buffers             = 32GB         # on the r7g.4xlarge, 128 GiB class
effective_cache_size       = 96GB
checkpoint_completion_target = 0.9
ssl                        = on
ssl_min_protocol_version   = 'TLSv1.3'
password_encryption        = scram-sha-256
log_min_duration_statement = 500ms

scope: pay-ledger
namespace: /pay/
name: pay_cac1_primary

etcd3:
  hosts:
    - etcd-cac1.pay.internal:2379
    - etcd-caw1.pay.internal:2379
    - etcd-onprem.pay.internal:2379      # witness, quorum only, no data

bootstrap:
  dcs:
    ttl: 30
    loop_wait: 10
    retry_timeout: 10
    synchronous_mode: true
    synchronous_mode_strict: true        # never degrade to async, preserve 0 RPO
    synchronous_node_count: 1
    postgresql:
      use_pg_rewind: true
      parameters:
        synchronous_commit: remote_apply
        synchronous_standby_names: 'ANY 1 (pay_caw1_standby)'

postgresql:
  listen: 0.0.0.0:5432
  data_dir: /pgdata/16
  authentication:
    replication: { username: replicator }
  callbacks:
    on_role_change: /opt/pay/on_role_change.sh   # updates the cell map on promotion

tags:
  nofailover: false
  clonefrom: true

{
  "version": 184,
  "cells": {
    "C1":  { "home": "ca-central-1", "writer": "pay_cac1_primary", "state": "active" },
    "C8":  { "home": "ca-central-1", "writer": "pay_cac1_primary", "state": "active" },
    "C9":  { "home": "ca-west-1",    "writer": "pay_caw1_primary", "state": "active" },
    "C16": { "home": "ca-west-1",    "writer": "pay_caw1_primary", "state": "active" }
  },
  "adoption": { "in_progress": false, "adopted_by": null }
}

resource "aws_rds_global_cluster" "rails" {
  global_cluster_identifier = "pay-rails"
  engine                    = "aurora-postgresql"
  engine_version            = "16.4"
  storage_encrypted         = true
}

resource "aws_rds_cluster" "rails_primary" {
  cluster_identifier        = "pay-rails-cac1"
  engine                    = "aurora-postgresql"
  engine_version            = "16.4"
  global_cluster_identifier = aws_rds_global_cluster.rails.id
  db_subnet_group_name      = aws_db_subnet_group.data.name
  vpc_security_group_ids    = [aws_security_group.aurora.id]
  kms_key_id                = aws_kms_key.rds.arn
  storage_encrypted         = true
  backup_retention_period   = 35
  iam_database_authentication_enabled = true
  db_cluster_parameter_group_name     = aws_rds_cluster_parameter_group.rails.name
}

resource "aws_rds_cluster_instance" "rails_primary" {
  count              = 2                       # writer plus reader, Multi-AZ
  cluster_identifier = aws_rds_cluster.rails_primary.id
  instance_class     = "db.r7g.2xlarge"        # Graviton
  engine             = "aurora-postgresql"
}

# Secondary region cluster, asynchronous replica, promotable on failover
resource "aws_rds_cluster" "rails_secondary" {
  provider                  = aws.caw1
  cluster_identifier        = "pay-rails-caw1"
  engine                    = "aurora-postgresql"
  engine_version            = "16.4"
  global_cluster_identifier = aws_rds_global_cluster.rails.id
  kms_key_id                = aws_kms_key.rds_caw1.arn
  depends_on                = [aws_rds_cluster_instance.rails_primary]
}

resource "aws_msk_cluster" "events" {
  cluster_name           = "pay-events"
  kafka_version          = "3.7.x"
  number_of_broker_nodes = 3
  broker_node_group_info {
    instance_type   = "kafka.m7g.large"
    client_subnets  = var.data_subnet_ids
    security_groups = [aws_security_group.msk.id]
    storage_info { ebs_storage_info { volume_size = 1000 } }
  }
  encryption_info {
    encryption_at_rest_kms_key_arn = aws_kms_key.msk.arn
    encryption_in_transit { client_broker = "TLS"  in_cluster = true }
  }
  client_authentication { sasl { iam = true } }
  open_monitoring { prometheus { jmx_exporter { enabled_in_broker = true } node_exporter { enabled_in_broker = true } } }
}

resource "aws_msk_replicator" "to_caw1" {
  replicator_name = "pay-events-to-caw1"
  kafka_cluster {
    amazon_msk_cluster { msk_cluster_arn = aws_msk_cluster.events.arn }
    vpc_config { subnet_ids = var.data_subnet_ids  security_group_ids = [aws_security_group.msk.id] }
  }
  kafka_cluster {
    amazon_msk_cluster { msk_cluster_arn = var.caw1_msk_arn }
    vpc_config { subnet_ids = var.caw1_data_subnet_ids  security_group_ids = [var.caw1_msk_sg] }
  }
  replication_info_list {
    target_compression_type = "GZIP"
    topic_replication { topic_name_configuration_sync_ms = 60000  copy_topic_configurations = true }
    consumer_group_replication { consumer_groups_to_replicate = ["*"] }
  }
}

resource "aws_elasticache_replication_group" "cache" {
  replication_group_id       = "pay-cache-cac1"
  description                = "payments cache, primary"
  engine                     = "redis"
  engine_version             = "7.1"
  node_type                  = "cache.r7g.large"
  num_node_groups            = 3            # cluster mode, sharded
  replicas_per_node_group    = 1
  automatic_failover_enabled = true
  multi_az_enabled           = true
  transit_encryption_enabled = true
  at_rest_encryption_enabled = true
  kms_key_id                 = aws_kms_key.redis.arn
  subnet_group_name          = aws_elasticache_subnet_group.data.name
  security_group_ids         = [aws_security_group.redis.id]
}

resource "aws_elasticache_global_replication_group" "cache" {
  global_replication_group_id_suffix = "pay-cache"
  primary_replication_group_id       = aws_elasticache_replication_group.cache.id
}

resource "aws_eks_pod_identity_association" "auth" {
  cluster_name    = aws_eks_cluster.this.name
  namespace       = "pay-auth"
  service_account = "payment-auth"
  role_arn        = aws_iam_role.payment_auth.arn
}

resource "aws_iam_role" "payment_auth" {
  name = "${local.name_prefix}-payment-auth"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Effect = "Allow"
      Principal = { Service = "pods.eks.amazonaws.com" }
      Action = ["sts:AssumeRole", "sts:TagSession"]
    }]
  })
}

resource "aws_iam_role_policy" "payment_auth" {
  role = aws_iam_role.payment_auth.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Sid    = "ReadOwnSecret"
        Effect = "Allow"
        Action = ["secretsmanager:GetSecretValue"]
        Resource = "arn:aws:secretsmanager:ca-central-1:ACCOUNT:secret:pay/prod/ledger/*"
      },
      {
        Sid    = "DecryptWithTenantKey"
        Effect = "Allow"
        Action = ["kms:Decrypt"]
        Resource = aws_kms_key.tenant_auth.arn
      }
    ]
  })
}

resource "aws_kms_key" "tenant_auth" {
  description             = "pay tenant key, payment-auth"
  enable_key_rotation     = true
  rotation_period_in_days = 90
  deletion_window_in_days = 30
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Sid = "RootAdmin"
        Effect = "Allow"
        Principal = { AWS = "arn:aws:iam::ACCOUNT:root" }
        Action = ["kms:Create*","kms:Describe*","kms:Enable*","kms:List*","kms:Put*","kms:ScheduleKeyDeletion"]
        Resource = "*"
      },
      {
        Sid = "WorkloadUse"
        Effect = "Allow"
        Principal = { AWS = aws_iam_role.payment_auth.arn }
        Action = ["kms:Decrypt","kms:GenerateDataKey"]
        Resource = "*"
      }
    ]
  })
}

# The cosign signing key is asymmetric on the elliptic curve, used by CI to sign and by Kyverno to verify
resource "aws_kms_key" "cosign" {
  description              = "pay image signing"
  customer_master_key_spec = "ECC_NIST_P256"
  key_usage                = "SIGN_VERIFY"
  enable_key_rotation      = false       # asymmetric keys are not auto-rotated
}
resource "aws_kms_alias" "cosign" { name = "alias/pay-cosign"  target_key_id = aws_kms_key.cosign.id }

driver: { kind: modern_ebpf }      # no kernel module, eBPF probe
falco:
  rules_files:
    - /etc/falco/falco_rules.yaml
    - /etc/falco/pay_rules.yaml
  json_output: true
customRules:
  pay_rules.yaml: |-
    - rule: Shell in cardholder container
      desc: A shell was started inside a CDE workload
      condition: spawned_process and container and proc.name in (bash, sh, zsh)
                 and k8s.ns.name in (pay-auth, pay-settlement)
      output: "Shell in CDE (pod=%k8s.pod.name ns=%k8s.ns.name cmd=%proc.cmdline)"
      priority: CRITICAL
falcosidekick:
  enabled: true
  config: { aws: { sns: { topicarn: "arn:aws:sns:ca-central-1:ACCOUNT:pay-security" } } }

resource "aws_iam_role" "ci_build" {
  name = "pay-ci-build"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Effect = "Allow"
      Principal = { Federated = "arn:aws:iam::ACCOUNT:oidc-provider/token.actions.githubusercontent.com" }
      Action = "sts:AssumeRoleWithWebIdentity"
      Condition = {
        StringEquals = { "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" }
        StringLike   = { "token.actions.githubusercontent.com:sub" = "repo:payments-org/pay-services:ref:refs/heads/main" }
      }
    }]
  })
}

name: build-sign-push
on:
  push: { branches: [main] }
  pull_request: {}
permissions:
  id-token: write        # required for OIDC
  contents: read
env:
  ECR: ACCOUNT.dkr.ecr.ca-central-1.amazonaws.com/pay/payment-auth
jobs:
  build:
    runs-on: [self-hosted, linux, arm64]    # private Fargate runners, see prior reference
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::ACCOUNT:role/pay-ci-build
          aws-region: ca-central-1
      - uses: aws-actions/amazon-ecr-login@v2
      - name: Unit and contract tests
        run: make test
      - name: Static analysis (SAST)
        run: semgrep ci --error
      - name: Build with rootless BuildKit
        id: build
        uses: docker/build-push-action@v6
        with:
          platforms: linux/arm64
          push: true
          tags: ${{ env.ECR }}:${{ github.sha }}
          provenance: true
          sbom: true
      - name: Generate SBOM
        uses: anchore/sbom-action@v0
        with: { image: "${{ env.ECR }}@${{ steps.build.outputs.digest }}", format: cyclonedx-json }
      - name: Vulnerability scan (fail on high)
        uses: aquasecurity/trivy-action@0.24.0
        with:
          image-ref: "${{ env.ECR }}@${{ steps.build.outputs.digest }}"
          severity: HIGH,CRITICAL
          exit-code: "1"
      - name: Sign image with cosign and KMS
        run: |
          cosign sign --yes --key awskms:///alias/pay-cosign \
            ${{ env.ECR }}@${{ steps.build.outputs.digest }}
      - name: Attest the SBOM
        run: |
          cosign attest --yes --key awskms:///alias/pay-cosign \
            --predicate sbom.cdx.json --type cyclonedx \
            ${{ env.ECR }}@${{ steps.build.outputs.digest }}
      - name: Record digest for delivery
        run: echo "${{ steps.build.outputs.digest }}" > image-digest.txt
      - uses: actions/upload-artifact@v4
        with: { name: image-digest, path: image-digest.txt }

name: promote-digest
on:
  workflow_run: { workflows: [build-sign-push], types: [completed], branches: [main] }
jobs:
  bump:
    if: ${{ github.event.workflow_run.conclusion == 'success' }}
    runs-on: [self-hosted, linux, arm64]
    steps:
      - uses: actions/checkout@v4
        with: { repository: payments-org/pay-config, ssh-key: ${{ secrets.CONFIG_DEPLOY_KEY }} }
      - uses: actions/download-artifact@v4
        with: { name: image-digest }
      - name: Update the non-production overlay first
        run: |
          DIGEST=$(cat image-digest.txt)
          cd overlays/nonprod
          kustomize edit set image pay/payment-auth=ACCOUNT.dkr.ecr.ca-central-1.amazonaws.com/pay/payment-auth@${DIGEST}
      - uses: peter-evans/create-pull-request@v6
        with:
          branch: promote/payment-auth
          title: "Promote payment-auth to ${{ github.sha }}"
          labels: automated, promotion

apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata: { name: payments, namespace: argocd }
spec:
  sourceRepos: ["https://github.com/payments-org/pay-config.git"]
  destinations:
    - { server: "https://kubernetes.default.svc", namespace: "pay-*" }
    - { name: "prod-caw1", namespace: "pay-*" }
  clusterResourceWhitelist: [ { group: "", kind: Namespace } ]
  roles:
    - name: deployer
      policies: ["p, proj:payments:deployer, applications, sync, payments/*, allow"]
---
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata: { name: payment-auth, namespace: argocd }
spec:
  generators:
    - list:
        elements:
          - { env: prod, region: cac1, cluster: "https://kubernetes.default.svc", wave: "2" }
          - { env: prod, region: caw1, cluster: "prod-caw1",                      wave: "3" }
  template:
    metadata:
      name: "payment-auth-{{env}}-{{region}}"
      annotations: { argocd.argoproj.io/sync-wave: "{{wave}}" }
    spec:
      project: payments
      source:
        repoURL: "https://github.com/payments-org/pay-config.git"
        targetRevision: main
        path: "overlays/{{env}}-{{region}}"
      destination: { server: "{{cluster}}", namespace: pay-auth }
      syncPolicy:
        automated: { prune: true, selfHeal: true }
        retry: { limit: 5 }

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata: { name: payment-auth, namespace: pay-auth }
spec:
  strategy:
    canary:
      canaryService: payment-auth-canary
      stableService: payment-auth-stable
      trafficRouting: { plugins: { argoproj-labs/gatewayAPI: { httpRoute: auth, namespace: pay-auth } } }
      steps:
        - setWeight: 5
        - pause: { duration: 5m }
        - analysis: { templates: [ { templateName: canary-health } ] }
        - setWeight: 25
        - pause: { duration: 10m }
        - analysis: { templates: [ { templateName: canary-health } ] }
        - setWeight: 50
        - pause: { duration: 10m }
---
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata: { name: canary-health, namespace: pay-auth }
spec:
  metrics:
    - name: success-rate
      interval: 1m
      successCondition: result[0] >= 0.999
      failureLimit: 1
      provider:
        prometheus:
          address: http://thanos-query.monitoring:9090
          query: |
            sum(rate(http_requests_total{app="payment-auth",code!~"5.."}[2m]))
            / sum(rate(http_requests_total{app="payment-auth"}[2m]))
    - name: p99-latency
      interval: 1m
      successCondition: result[0] <= 0.250
      provider:
        prometheus:
          address: http://thanos-query.monitoring:9090
          query: |
            histogram_quantile(0.99,
              sum(rate(http_request_duration_seconds_bucket{app="payment-auth"}[2m])) by (le))

# thanos object store, per region, S3 with the regional KMS key
type: S3
config:
  bucket: pay-thanos-cac1
  endpoint: s3.ca-central-1.amazonaws.com
  sse_config: { type: SSE-KMS, kms_key_id: "alias/pay-observability" }
---
# OpenTelemetry collector, traces and metrics
receivers:
  otlp: { protocols: { grpc: {}, http: {} } }
processors:
  batch: {}
  memory_limiter: { check_interval: 1s, limit_percentage: 80 }
exporters:
  prometheusremotewrite: { endpoint: "http://prometheus.monitoring:9090/api/v1/write" }
  otlphttp/tempo: { endpoint: "http://tempo.monitoring:4318" }
service:
  pipelines:
    traces:  { receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlphttp/tempo] }
    metrics: { receivers: [otlp], processors: [memory_limiter, batch], exporters: [prometheusremotewrite] }

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata: { name: pay-auth-slo, namespace: monitoring }
spec:
  groups:
    - name: payment-auth.slo
      rules:
        - record: job:auth_request_error:ratio_rate5m
          expr: |
            sum(rate(http_requests_total{app="payment-auth",code=~"5.."}[5m]))
            / sum(rate(http_requests_total{app="payment-auth"}[5m]))
        - alert: AuthErrorBudgetFastBurn
          # 99.999 percent objective, page when burning the budget 14x over a short window
          expr: |
            job:auth_request_error:ratio_rate5m > (14.4 * 0.00001)
            and ignoring(window)
            job:auth_request_error:ratio_rate1h > (14.4 * 0.00001)
          for: 2m
          labels: { severity: page }
          annotations: { summary: "Authorization error budget fast burn" }

resource "aws_fis_experiment_template" "ledger_primary_loss" {
  description = "Terminate the ledger primary and verify zero data loss failover"
  role_arn    = aws_iam_role.fis.arn

  stop_condition {
    source = "aws:cloudwatch:alarm"
    value  = aws_cloudwatch_metric_alarm.auth_error_budget.arn   # abort if customers are harmed
  }
  action {
    name      = "stop-primary"
    action_id = "aws:ec2:stop-instances"
    target { key = "Instances" value = "ledger-primary" }
  }
  target {
    name           = "ledger-primary"
    resource_type  = "aws:ec2:instance"
    selection_mode = "ALL"
    resource_tag { key = "Role" value = "ledger-primary" }
  }
  experiment_options { account_targeting = "single-account" empty_target_resolution_mode = "fail" }
}