{"templateId":"markdown","versions":[{"version":"v1.40","label":"v1.40 STS","link":"/products/custody/deployment/integrate-kms/production-best-practices","default":true,"active":false,"folderId":"c15a2701"},{"version":"v1.39","label":"v1.39 STS","link":"/products/custody/v1.39/deployment/integrate-kms/production-best-practices","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.38","label":"v1.38 STS","link":"/products/custody/v1.38/deployment/integrate-kms/production-best-practices","default":false,"active":true,"folderId":"c15a2701"},{"version":"v1.34","label":"v1.34 LTS","link":"/products/custody/v1.34/deployment/integrate-kms/production-best-practices","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.26","label":"v1.26 LTS","link":"/products/custody/v1.26/deployment/integrate-kms/production-best-practices","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.19","label":"v1.19 LTS","link":"/products/custody/v1.19/deployment/integrate-kms/production-best-practices","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.15","label":"v1.15 LTS","link":"/products/custody/v1.15/deployment/integrate-kms/production-best-practices","default":false,"active":false,"folderId":"c15a2701"}],"sharedDataIds":{"sidebar":"sidebar-products/custody/@v1.15/sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition","tabs","tab"]},"type":"markdown"},"seo":{"title":"Production best practices","description":"User guides, API reference, and support resources.","siteUrl":"https://docs.ripple.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"production-best-practices","__idx":0},"children":["Production best practices"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This guide covers production-ready deployment patterns for ","Ripple Custody"," with any KMS platform (HSM, Cloud KMS, or MPC). These patterns ensure high availability, observability, and operational resilience."]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Applies to"]},": All KMS integrations — both HSM and MPC deployments."]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"namespace-segmentation","__idx":1},"children":["Namespace segmentation"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For production deployments, use ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["namespace segmentation"]}," to isolate different components and enable multi-tenancy."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"recommended-namespace-structure","__idx":2},"children":["Recommended namespace structure"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Namespace"},"children":["Namespace"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Purpose"},"children":["Purpose"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Components"},"children":["Components"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["custody-core"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Core infrastructure"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Notary, Core services"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["custody-vaults"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Vault workloads"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Vault instances"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["custody-networks"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Network adapters"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Blockchain indexers, network bridges"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["custody-monitoring"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Observability"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Prometheus, Grafana, alerting"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"benefits","__idx":3},"children":["Benefits"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Security isolation"]},": Network policies restrict cross-namespace traffic."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Resource quotas"]},": Limit resource consumption per namespace."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["RBAC Boundaries"]},": Fine-grained access control per component"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Operational clarity"]},": Clear separation of concerns."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Compliance"]},": Meet regulatory requirements for multi-tenant isolation"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"implementation","__idx":4},"children":["Implementation"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Create namespaces\nkubectl create namespace custody-core\nkubectl create namespace custody-vaults\nkubectl create namespace custody-networks\nkubectl create namespace custody-monitoring\n\n# Apply network policies (restrict cross-namespace traffic)\nkubectl apply -f network-policies/\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Helm deployment"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"helm upgrade --install harmonize ./harmonize-custody \\\n --namespace custody-core \\\n --set components.notary.namespace=custody-core \\\n --set harmonize.vaults.*.namespace=custody-vaults \\\n -f production.yaml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"high-availability","__idx":5},"children":["High availability"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"kubernetes-components","__idx":6},"children":["Kubernetes components"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Deploy multiple replicas of critical components. Note that the notary must run as a singleton (Merkle tree operations cannot be parallelized), while the vault supports multiple replicas for high availability:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# production.yaml\nharmonize:\n vaults:\n \"00000000-0000-0000-0000-000000000000\":\n enabled: true\n replicas: 3 # Multiple vault replicas for HA\n\n # Pod anti-affinity (spread across nodes)\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n app: harmonize-vault\n topologyKey: kubernetes.io/hostname\n\n resources:\n limits:\n cpu: 2000m\n memory: 512Mi\n requests:\n cpu: 500m\n memory: 256Mi\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"kms-high-availability-ha","__idx":7},"children":["KMS high availability (HA)"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"KMS Platform"},"children":["KMS Platform"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"HA Strategy"},"children":["HA Strategy"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Thales Luna HSM"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["HA Group with multiple HSMs, automatic failover"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["BlockSafe HSM"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["HSM clustering across racks/locations"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Securosys Primus HSM"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["HA Group with multiple HSMs, automatic failover"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["AWS CloudHSM"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Multi-AZ cluster deployment"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["IBM HSM"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["GREP11 API with IBM HPVS, Crypto Express (CEX) adapter clustering for secure components only"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["MPC"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Distributed nodes (3-of-4 threshold signing)"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Refer to your specific KMS integration guide for detailed HA configuration."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"environment-topologies-prod-vs-dev","__idx":8},"children":["Environment topologies: PROD vs DEV"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Use different HSM topologies for production and non-production environments to balance security with cost efficiency."]},{"$$mdtype":"Tag","name":"Tabs","attributes":{"size":"medium"},"children":[{"$$mdtype":"Tag","name":"TabItemFragment","attributes":{"label":"Production (HA cluster)","disable":false},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Production"]}," environments require a full HA cluster with partition replication across multiple HSMs:"]},{"$$mdtype":"Tag","name":"Diagram","attributes":{"data-language":"mermaid","diagramType":"mermaid","diagramSource":"flowchart TB\n subgraph prod[\"PROD Environment - HA Cluster\"]\n subgraph apps[\"Client Applications\"]\n direction LR\n notary[\"Notary\"]\n vault1[\"Vault Replica\"]\n vault2[\"Vault Replica\"]\n end\n\n subgraph luna[\"Thales Luna HA Group\"]\n direction LR\n hsm1[\"HSM 1\"]\n hsm2[\"HSM 2\"]\n hsm3[\"HSM 3\"]\n\n subgraph p1[\"Partition\"]\n part1[\" \"]\n end\n subgraph p2[\"Partition\"]\n part2[\" \"]\n end\n subgraph p3[\"Partition\"]\n part3[\" \"]\n end\n\n hsm1 --- p1\n hsm2 --- p2\n hsm3 --- p3\n p1 <-->|\"Replication\"| p2\n p2 <-->|\"Replication\"| p3\n end\n end\n\n notary --> luna\n vault1 --> luna\n vault2 --> luna\n","diagramHtml":"

PROD Environment - HA Cluster

Thales Luna HA Group

Replication

Replication

Partition

HSM 1

Partition

HSM 2

Partition

HSM 3

Client Applications

Notary

Vault Replica

Vault Replica

","diagramHtmlDark":"

PROD Environment - HA Cluster

Thales Luna HA Group

Replication

Replication

Partition

HSM 1

Partition

HSM 2

Partition

HSM 3

Client Applications

Notary

Vault Replica

Vault Replica

"},"children":["flowchart TB\n subgraph prod[\"PROD Environment - HA Cluster\"]\n subgraph apps[\"Client Applications\"]\n direction LR\n notary[\"Notary\"]\n vault1[\"Vault Replica\"]\n vault2[\"Vault Replica\"]\n end\n\n subgraph luna[\"Thales Luna HA Group\"]\n direction LR\n hsm1[\"HSM 1\"]\n hsm2[\"HSM 2\"]\n hsm3[\"HSM 3\"]\n\n subgraph p1[\"Partition\"]\n part1[\" \"]\n end\n subgraph p2[\"Partition\"]\n part2[\" \"]\n end\n subgraph p3[\"Partition\"]\n part3[\" \"]\n end\n\n hsm1 --- p1\n hsm2 --- p2\n hsm3 --- p3\n p1 <-->|\"Replication\"| p2\n p2 <-->|\"Replication\"| p3\n end\n end\n\n notary --> luna\n vault1 --> luna\n vault2 --> luna\n"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Key points:"]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["3+ HSMs in an HA group for redundancy"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Partition replication across all HSMs"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Multiple vault replicas for signing capacity"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Automatic failover if any HSM becomes unavailable"]}]}]},{"$$mdtype":"Tag","name":"TabItemFragment","attributes":{"label":"DEV/UAT (single HSM)","disable":false},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Non-production"]}," environments can use a single HSM with multiple partitions to reduce costs:"]},{"$$mdtype":"Tag","name":"Diagram","attributes":{"data-language":"mermaid","diagramType":"mermaid","diagramSource":"flowchart TB\n subgraph dev[\"DEV Environment - Single HSM\"]\n subgraph apps1[\"DEV Applications\"]\n notary1[\"Notary\"]\n vault1[\"Vault\"]\n end\n\n subgraph apps2[\"UAT Applications\"]\n notary2[\"Notary\"]\n vault2[\"Vault\"]\n end\n\n subgraph luna[\"Thales Luna HSM\"]\n partDev[\"Partition: DEV\"]\n partUat[\"Partition: UAT\"]\n end\n end\n\n notary1 --> partDev\n vault1 --> partDev\n notary2 --> partUat\n vault2 --> partUat\n","diagramHtml":"

DEV Environment - Single HSM

Thales Luna HSM

UAT Applications

DEV Applications

Partition: DEV

Notary

Vault

Notary

Partition: UAT

Vault

","diagramHtmlDark":"

DEV Environment - Single HSM

Thales Luna HSM

UAT Applications

DEV Applications

Partition: DEV

Notary

Vault

Notary

Partition: UAT

Vault

"},"children":["flowchart TB\n subgraph dev[\"DEV Environment - Single HSM\"]\n subgraph apps1[\"DEV Applications\"]\n notary1[\"Notary\"]\n vault1[\"Vault\"]\n end\n\n subgraph apps2[\"UAT Applications\"]\n notary2[\"Notary\"]\n vault2[\"Vault\"]\n end\n\n subgraph luna[\"Thales Luna HSM\"]\n partDev[\"Partition: DEV\"]\n partUat[\"Partition: UAT\"]\n end\n end\n\n notary1 --> partDev\n vault1 --> partDev\n notary2 --> partUat\n vault2 --> partUat\n"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Key points:"]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Single HSM with separate partitions per environment"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["DEV and UAT isolated via partition boundaries"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Cost-effective for non-production workloads"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["No HA - acceptable for development/testing"]}]}]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"monitoring-and-alerting","__idx":9},"children":["Monitoring and alerting"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"key-metrics","__idx":10},"children":["Key metrics"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Category"},"children":["Category"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Metrics to Monitor"},"children":["Metrics to Monitor"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["KMS/HSM Health"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Connection status, latency, error rate"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Signing Operations"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Requests/second, success rate, latency"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Pod Health"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["CPU, memory, restarts, readiness"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Application"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Transaction throughput, queue depth"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"prometheus-servicemonitor","__idx":11},"children":["Prometheus ServiceMonitor"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"apiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n name: harmonize-notary\n namespace: custody-monitoring\nspec:\n selector:\n matchLabels:\n app: harmonize-notary\n namespaceSelector:\n matchNames:\n - custody-core\n endpoints:\n - port: metrics\n interval: 30s\n path: /metrics\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"recommended-alerts","__idx":12},"children":["Recommended alerts"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Alert"},"children":["Alert"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Condition"},"children":["Condition"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Severity"},"children":["Severity"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["KMSConnectionFailed"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["HSM/KMS connection down > 1 min"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Critical"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["SigningLatencyHigh"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["p99 latency > 500ms for 5 min"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Warning"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["NotaryPodDown"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Notary pod unavailable > 1 min"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Critical"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["VaultReplicasLow"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Vault replicas < 2"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Warning"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["HighErrorRate"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Error rate > 1% for 5 min"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Warning"]}]}]}]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"certificate-rotation","__idx":13},"children":["Certificate rotation"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For mTLS-enabled deployments (Luna, BlockSafe):"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"rotation-procedure","__idx":14},"children":["Rotation procedure"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Generate new certificates"]}," using your HSM vendor tools"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create new Kubernetes secret"]}," with updated certificates"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Perform rolling update"]}," via Helm to pick up new credentials"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Verify connectivity"]}," to HSM with new certificates"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Revoke old certificates"]}," after confirming new ones work"]}]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Zero-Downtime Rotation"]},": With multiple vault replicas, rolling updates ensure continuous availability during certificate rotation. Note that the notary runs as a singleton, so plan a brief maintenance window for notary certificate rotation."]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"backup-and-disaster-recovery","__idx":15},"children":["Backup and disaster recovery"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"what-to-backup","__idx":16},"children":["What to backup"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Component"},"children":["Component"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Backup Method"},"children":["Backup Method"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Frequency"},"children":["Frequency"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["HSM Keys"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Vendor-specific backup (Luna: partition backup, AWS CloudHSM: cluster backup)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Daily"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Kubernetes Secrets"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Encrypted backup to secure storage"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Daily"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Helm Values"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Version control (Git)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["On change"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Configuration"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Infrastructure as Code (Terraform/CDK)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["On change"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"recovery-testing","__idx":17},"children":["Recovery testing"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Perform quarterly disaster recovery testing:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["HSM Recovery"]},": Restore keys to test partition/cluster"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Kubernetes recovery"]},": Redeploy from IaC to test cluster."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["End-to-End Test"]},": Verify signing operations work after recovery"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Document results"]},": Update runbooks based on findings."]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"related-guides","__idx":18},"children":["Related guides"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/overview"},"children":["KMS integration overview"]}," - All supported KMS platforms"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/advanced-secret-management"},"children":["Advanced secret management"]}," - ESO, Vault Agent, AWS Secrets Manager"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/governance/domains/inject-a-domain"},"children":["Domain injection"]}," - Advanced key isolation patterns"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/on-premise-hsm/thales-luna"},"children":["Thales Luna HSM"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/on-premise-hsm/blocksafe"},"children":["BlockSafe HSM"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/cloud-hsm/aws-cloudhsm"},"children":["AWS CloudHSM"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/on-premise-hsm/ibm-linuxone"},"children":["IBM LinuxONE"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/mpc/overview"},"children":["MPC overview"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/v1.38/deployment/integrate-kms/mpc/backups"},"children":["MPC backups"]}]}]}]},"headings":[{"value":"Production best practices","id":"production-best-practices","depth":1},{"value":"Namespace segmentation","id":"namespace-segmentation","depth":2},{"value":"Recommended namespace structure","id":"recommended-namespace-structure","depth":3},{"value":"Benefits","id":"benefits","depth":3},{"value":"Implementation","id":"implementation","depth":3},{"value":"High availability","id":"high-availability","depth":2},{"value":"Kubernetes components","id":"kubernetes-components","depth":3},{"value":"KMS high availability (HA)","id":"kms-high-availability-ha","depth":3},{"value":"Environment topologies: PROD vs DEV","id":"environment-topologies-prod-vs-dev","depth":3},{"value":"Monitoring and alerting","id":"monitoring-and-alerting","depth":2},{"value":"Key metrics","id":"key-metrics","depth":3},{"value":"Prometheus ServiceMonitor","id":"prometheus-servicemonitor","depth":3},{"value":"Recommended alerts","id":"recommended-alerts","depth":3},{"value":"Certificate rotation","id":"certificate-rotation","depth":2},{"value":"Rotation procedure","id":"rotation-procedure","depth":3},{"value":"Backup and disaster recovery","id":"backup-and-disaster-recovery","depth":2},{"value":"What to backup","id":"what-to-backup","depth":3},{"value":"Recovery testing","id":"recovery-testing","depth":3},{"value":"Related guides","id":"related-guides","depth":2}],"frontmatter":{"seo":{"title":"Production best practices"}},"lastModified":"2026-07-27T10:10:43.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/products/custody/v1.38/deployment/integrate-kms/production-best-practices","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}