{"templateId":"markdown","versions":[{"version":"v1.39","label":"v1.39 STS","link":"/products/custody/deployment/integrate-kms/advanced-secret-management","default":true,"active":true,"folderId":"c15a2701"},{"version":"v1.38","label":"v1.38 STS","link":"/products/custody/v1.38/deployment/integrate-kms/advanced-secret-management","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.34","label":"v1.34 LTS","link":"/products/custody/v1.34/deployment/integrate-kms/advanced-secret-management","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.26","label":"v1.26 LTS","link":"/products/custody/v1.26/deployment/integrate-kms/advanced-secret-management","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.19","label":"v1.19 LTS","link":"/products/custody/v1.19/deployment/integrate-kms/advanced-secret-management","default":false,"active":false,"folderId":"c15a2701"},{"version":"v1.15","label":"v1.15 LTS","link":"/products/custody/v1.15/deployment/integrate-kms/advanced-secret-management","default":false,"active":false,"folderId":"c15a2701"}],"sharedDataIds":{"sidebar":"sidebar-products/custody/@v1.15/sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition"]},"type":"markdown"},"seo":{"title":"Advanced secret management","description":"User guides, API reference, and support resources.","siteUrl":"https://docs.ripple.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"advanced-secret-management","__idx":0},"children":["Advanced secret management"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This guide covers advanced secret management patterns for ","Ripple Custody",", including ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["External Secrets Operator (ESO)"]},", ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["HashiCorp Vault Agent Injector"]},", and ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["AWS Secrets Manager"]}," integration."]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Target audience"]},": This guide is intended for platform engineers and DevOps teams managing production ","Ripple Custody"," deployments with enterprise secret management requirements."]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"table-of-contents","__idx":1},"children":["Table of contents"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#overview"},"children":["Overview"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#what-is-advanced-secret-management"},"children":["Why advanced secret management"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#external-secrets-operator-eso"},"children":["External Secrets Operator (ESO)"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#hashicorp-vault-agent-injector"},"children":["HashiCorp Vault Agent Injector"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#aws-secrets-manager"},"children":["AWS Secrets Manager"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#comparison-eso-vs-vault-injector-vs-aws-secrets-manager"},"children":["Comparison: ESO vs Vault Injector vs AWS Secrets Manager"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#security-best-practices"},"children":["Security best practices"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#troubleshooting"},"children":["Troubleshooting"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#platform-specific-examples"},"children":["Platform-specific examples"]}]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"overview","__idx":2},"children":["Overview"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"what-is-advanced-secret-management","__idx":3},"children":["What is advanced secret management?"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Advanced secret management involves using external secret stores (like HashiCorp Vault or AWS Secrets Manager) to centrally manage sensitive credentials, with automatic synchronization to Kubernetes Secrets or direct injection into application pods."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"benefits","__idx":4},"children":["Benefits"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Centralized secret management"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Single source of truth for all secrets"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Centralized access control and audit logging"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Consistent secret rotation policies"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Automatic rotation"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Secrets are automatically updated in Kubernetes"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["No manual intervention required"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Reduced risk of stale credentials"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Enhanced security"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Secrets never stored in Git or Helm values"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Fine-grained access control via IAM/RBAC"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Encryption at rest and in transit"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Compliance"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Audit trail for all secret access"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Meets regulatory requirements (SOC 2, PCI-DSS, etc.)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Separation of duties (platform team vs application team)"]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"external-secrets-operator-eso","__idx":5},"children":["External Secrets Operator (ESO)"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"overview-1","__idx":6},"children":["Overview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["External Secrets Operator (ESO)"]}," synchronizes secrets from external secret stores (HashiCorp Vault, AWS Secrets Manager, Azure Key Vault) to Kubernetes Secrets."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["How it works"]},":"]},{"$$mdtype":"Tag","name":"Diagram","attributes":{"data-language":"mermaid","diagramType":"mermaid","diagramSource":"flowchart LR\n vault[\"HashiCorp Vault
secret/app/luna-creds\"]\n es[\"ExternalSecret\"]\n ks[\"Kubernetes Secret\"]\n pod[\"Application Pod\"]\n\n vault -->|ESO polls| es\n es -->|creates| ks\n ks -->|volumeMount| pod\n","diagramHtml":"

ESO polls

creates

volumeMount

HashiCorp Vault
secret/app/luna-creds

ExternalSecret

Kubernetes Secret

Application Pod

","diagramHtmlDark":"

ESO polls

creates

volumeMount

HashiCorp Vault
secret/app/luna-creds

ExternalSecret

Kubernetes Secret

Application Pod

"},"children":["flowchart LR\n vault[\"HashiCorp Vault
secret/app/luna-creds\"]\n es[\"ExternalSecret\"]\n ks[\"Kubernetes Secret\"]\n pod[\"Application Pod\"]\n\n vault -->|ESO polls| es\n es -->|creates| ks\n ks -->|volumeMount| pod\n"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"prerequisites","__idx":7},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Install External Secrets Operator"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Add Helm repository\nhelm repo add external-secrets https://charts.external-secrets.io\nhelm repo update\n\n# Install ESO\nhelm install external-secrets \\\n external-secrets/external-secrets \\\n --namespace external-secrets-system \\\n --create-namespace \\\n --set installCRDs=true\n\n# Verify installation\nkubectl get pods -n external-secrets-system\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Configure HashiCorp Vault"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Enable Kubernetes auth method\nvault auth enable kubernetes\n\n# Configure Kubernetes auth\nvault write auth/kubernetes/config \\\n kubernetes_host=\"https://kubernetes.default.svc:443\"\n\n# Create policy for ESO\nvault policy write eso-reader - < Kubernetes Secret key\n data:\n # ca.pem\n - secretKey: ca.pem\n remoteRef:\n key: app/luna-creds\n property: ca.pem\n\n # client.pem\n - secretKey: client.pem\n remoteRef:\n key: app/luna-creds\n property: client.pem\n\n # client.key\n - secretKey: client.key\n remoteRef:\n key: app/luna-creds\n property: client.key\n\n # PIN\n - secretKey: KMS_LUNA_PIN\n remoteRef:\n key: app/luna-creds\n property: pin\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Apply"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"kubectl apply -f externalsecret-luna.yaml\n\n# Verify ExternalSecret\nkubectl get externalsecret -n custody-core\n\n# Verify Kubernetes Secret was created\nkubectl get secret kms-luna-credentials -n custody-core\n\n# View secret keys (not values)\nkubectl describe secret kms-luna-credentials -n custody-core\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"step-3-reference-secret-in-helm-values","__idx":10},"children":["Step 3: Reference secret in Helm values"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Update your Helm values to reference the ESO-managed secret."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Example ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["values.yaml"]}]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# values.yaml\nkms:\n kind: \"luna\"\n\n # Reference existing secret created by ESO\n existingSecret: \"kms-luna-credentials\"\n\n # Optional: Specify custom key names if different from defaults\n existingSecretKeys:\n pin: \"KMS_LUNA_PIN\"\n caPem: \"ca.pem\"\n clientPem: \"client.pem\"\n clientKey: \"client.key\"\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Deploy"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"helm upgrade --install harmonize ./harmonize-custody \\\n --namespace custody-core \\\n -f values.yaml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"step-4-verify-secret-rotation","__idx":11},"children":["Step 4: Verify secret rotation"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["ESO automatically updates the Kubernetes Secret when the Vault secret changes."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Test rotation"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Update secret in Vault\nvault kv put secret/app/luna-creds \\\n ca.pem=@ca-new.pem \\\n client.pem=@client-new.pem \\\n client.key=@client-new.key \\\n pin=\"new-hsm-pin\"\n\n# Wait for refreshInterval (default: 1h) or force sync\nkubectl annotate externalsecret external-kms-luna-secret \\\n force-sync=$(date +%s) \\\n -n custody-core\n\n# Verify secret was updated\nkubectl get secret kms-luna-credentials -n custody-core -o yaml\n\n# Restart pods to pick up new secret\nkubectl rollout restart deployment/harmonize-notary -n custody-core\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"hashicorp-vault-agent-injector","__idx":12},"children":["HashiCorp Vault Agent Injector"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"overview-2","__idx":13},"children":["Overview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Vault Agent Injector"]}," uses a sidecar pattern to inject secrets directly into application pods without creating Kubernetes Secrets."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["How it works"]},":"]},{"$$mdtype":"Tag","name":"Diagram","attributes":{"data-language":"mermaid","diagramType":"mermaid","diagramSource":"flowchart LR\n agent[\"vault-agent sidecar\"]\n vol[\"/vault/secrets/\"]\n kms[\"kms-luna container\"]\n\n agent -->|writes secrets| vol\n vol -->|volumeMount| kms\n","diagramHtml":"

writes secrets

volumeMount

vault-agent sidecar

/vault/secrets/

kms-luna container

","diagramHtmlDark":"

writes secrets

volumeMount

vault-agent sidecar

/vault/secrets/

kms-luna container

"},"children":["flowchart LR\n agent[\"vault-agent sidecar\"]\n vol[\"/vault/secrets/\"]\n kms[\"kms-luna container\"]\n\n agent -->|writes secrets| vol\n vol -->|volumeMount| kms\n"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"prerequisites-1","__idx":14},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Install Vault Agent Injector"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Add Helm repository\nhelm repo add hashicorp https://helm.releases.hashicorp.com\nhelm repo update\n\n# Install Vault with injector\nhelm install vault hashicorp/vault \\\n --namespace vault-system \\\n --create-namespace \\\n --set \"injector.enabled=true\" \\\n --set \"server.dev.enabled=true\"\n\n# Verify installation\nkubectl get pods -n vault-system\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Configure Vault"]}," (same as ESO prerequisites)"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"step-1-annotate-pod-for-injection","__idx":15},"children":["Step 1: Annotate pod for injection"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Add Vault annotations to your Helm values to enable injection."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Example ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["values.yaml"]}]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# values.yaml\nkms:\n kind: \"luna\"\n\n # Vault Agent Injector annotations\n podAnnotations:\n vault.hashicorp.com/agent-inject: \"true\"\n vault.hashicorp.com/role: \"eso-reader\"\n vault.hashicorp.com/agent-inject-secret-ca.pem: \"secret/data/app/luna-creds\"\n vault.hashicorp.com/agent-inject-template-ca.pem: |\n {{- with secret \"secret/data/app/luna-creds\" -}}\n {{ .Data.data.ca.pem }}\n {{- end -}}\n\n vault.hashicorp.com/agent-inject-secret-client.pem: \"secret/data/app/luna-creds\"\n vault.hashicorp.com/agent-inject-template-client.pem: |\n {{- with secret \"secret/data/app/luna-creds\" -}}\n {{ .Data.data.client.pem }}\n {{- end -}}\n\n vault.hashicorp.com/agent-inject-secret-client.key: \"secret/data/app/luna-creds\"\n vault.hashicorp.com/agent-inject-template-client.key: |\n {{- with secret \"secret/data/app/luna-creds\" -}}\n {{ .Data.data.client.key }}\n {{- end -}}\n\n vault.hashicorp.com/agent-inject-secret-pin: \"secret/data/app/luna-creds\"\n vault.hashicorp.com/agent-inject-template-pin: |\n {{- with secret \"secret/data/app/luna-creds\" -}}\n {{ .Data.data.pin }}\n {{- end -}}\n\n # Environment variables pointing to injected files\n extraEnvVars:\n - name: KMS_LUNA_CA_PEM_PATH\n value: \"/vault/secrets/ca.pem\"\n - name: KMS_LUNA_CLIENT_PEM_PATH\n value: \"/vault/secrets/client.pem\"\n - name: KMS_LUNA_CLIENT_KEY_PATH\n value: \"/vault/secrets/client.key\"\n - name: KMS_LUNA_PIN\n valueFrom:\n secretKeyRef:\n name: vault-injected-pin\n key: pin\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"step-2-deploy-and-verify","__idx":16},"children":["Step 2: Deploy and verify"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Deploy with Vault Agent Injector\nhelm upgrade --install harmonize ./harmonize-custody \\\n --namespace custody-core \\\n -f values.yaml\n\n# Verify vault-agent sidecar was injected\nkubectl get pods -n custody-core\n\n# Expected output:\n# NAME READY STATUS RESTARTS AGE\n# harmonize-notary- 2/2 Running 0 1m\n# ^^^\n# 2 containers: kms-luna + vault-agent\n\n# Verify secrets were injected\nkubectl exec -it -n custody-core deployment/harmonize-notary -c kms-luna -- \\\n ls -la /vault/secrets/\n\n# Expected output:\n# -rw-r--r-- 1 vault vault 1234 Jun 26 10:00 ca.pem\n# -rw-r--r-- 1 vault vault 1234 Jun 26 10:00 client.pem\n# -rw-r--r-- 1 vault vault 1234 Jun 26 10:00 client.key\n# -rw-r--r-- 1 vault vault 12 Jun 26 10:00 pin\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"aws-secrets-manager","__idx":17},"children":["AWS Secrets Manager"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"overview-3","__idx":18},"children":["Overview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["AWS Secrets Manager"]}," is used by the Vault Deployer UI to store deployment configuration and HSM credentials for AWS Nitro enclave deployments."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Storage path structure"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"/ripple/custody/////\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Where:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":[""]},": Your ","Ripple Custody"," tenant identifier"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":[""]},": Your ","Ripple Custody"," instance identifier"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":[""]},": Always ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["vault"]}," for vault deployments"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":[""]},": The vault ID (UUID)"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"secrets-stored","__idx":19},"children":["Secrets stored"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["config"]}," key"]},": Deployment configuration"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{\n \"deployment\": {\n \"vault\": {\n \"release-name\": \"vault-01\",\n \"construct-id\": \"00000000-0000-0000-0000-000000000001\"\n }\n },\n \"custody\": {\n \"environment\": \"https://api.your-instance.custody.example.com/\"\n },\n \"infrastructure\": {\n \"aws\": {\n \"region\": \"us-east-1\",\n \"ec2-iam-role\": \"vault-role\",\n \"vpc\": {\n \"create-new\": false,\n \"id\": \"vpc-abc123\",\n \"subnet-id\": \"subnet-def456\"\n }\n }\n }\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["hsm"]}," key"]},": HSM credentials"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{\n \"kms-awscloudhsm\": \"cluster-a1b2c3d4e5f\",\n \"kms-awscloudhsm-cacert\": \"-----BEGIN CERTIFICATE-----\\nMI...==\\n-----END CERTIFICATE-----\\n\",\n \"kms-password\": \"CryptoUserPassword\",\n \"kms-soft-wrapping-key\": \"0123456789abcdef...\",\n \"kms-username\": \"vault-crypto-user\",\n \"kms-wrap-key\": \"0000000000000001\",\n \"kms-awscloudhsm-clientcert\": \"-----BEGIN CERTIFICATE-----\\n...\",\n \"kms-awscloudhsm-clientkey\": \"-----BEGIN PRIVATE KEY-----\\n...\"\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"security-best-practices","__idx":20},"children":["Security best practices"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Limit Access"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Create IAM policy for limited access\ncat > secrets-manager-policy.json <//vault//config \\\n --query SecretString \\\n --output text | jq .\n\n# Get HSM credentials\naws secretsmanager get-secret-value \\\n --secret-id /ripple/custody///vault//hsm \\\n --query SecretString \\\n --output text | jq .\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Important"]},": While AWS Secrets Manager is used during vault ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["deployment"]}," automation, ","Ripple Custody"," vault itself ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["does not"]}," use AWS Secrets Manager to store wallet private keys. Wallet keys are encrypted with AWS CloudHSM and stored in the ","Ripple Custody"," database."]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"comparison-eso-vs-vault-injector-vs-aws-secrets-manager","__idx":22},"children":["Comparison: ESO vs Vault Injector vs AWS Secrets Manager"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Feature"},"children":["Feature"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"External Secrets Operator"},"children":["External Secrets Operator"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Vault Agent Injector"},"children":["Vault Agent Injector"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"AWS Secrets Manager"},"children":["AWS Secrets Manager"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Secret Storage"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Kubernetes Secret"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Files in shared volume"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["AWS Secrets Manager"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Rotation"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Automatic (refreshInterval)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Automatic (lease renewal)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Manual or Lambda-based"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Kubernetes Native"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Yes (CRDs)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Yes (annotations)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["No (AWS-specific)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Multi-Cloud"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Yes (Vault, AWS, Azure)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Yes (Vault only)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["No (AWS only)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Complexity"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Low"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Medium"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Low"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Performance"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Good (cached in K8s Secret)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Excellent (direct injection)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Good (cached in AWS)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Security"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Good (K8s RBAC)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Excellent (no K8s Secret)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Good (IAM policies)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Use Case"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["General-purpose"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["High-security environments"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["AWS Nitro enclaves"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Ripple Custody"," Support"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Recommended"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Supported"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Vault Deployer UI only"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"recommendations","__idx":23},"children":["Recommendations"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Use External Secrets Operator (ESO) when"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You need multi-cloud support"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You want Kubernetes-native secret management"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You need automatic rotation with minimal complexity"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Recommended for most ","Ripple Custody"," deployments"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Use Vault Agent Injector when"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You have strict security requirements (no K8s Secrets)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You need fine-grained secret access control"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You want secrets to be ephemeral (not persisted in K8s)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You're already using HashiCorp Vault extensively"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Use AWS Secrets Manager when"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You're deploying vaults in AWS Nitro enclaves"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You're using the Vault Deployer UI"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You need AWS-native secret management"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Only for Vault Deployer UI deployments"]}]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"security-best-practices-1","__idx":24},"children":["Security best practices"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"principle-of-least-privilege","__idx":25},"children":["Principle of least privilege"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Vault Policies"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"hcl","header":{"controls":{"copy":{}}},"source":"# eso-reader-policy.hcl\n# Read-only access to specific paths\npath \"secret/data/app/luna-creds\" {\n capabilities = [\"read\"]\n}\n\n# Deny all other paths\npath \"secret/*\" {\n capabilities = [\"deny\"]\n}\n","lang":"hcl"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Kubernetes RBAC"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: secret-reader\n namespace: custody-core\nrules:\n- apiGroups: [\"\"]\n resources: [\"secrets\"]\n resourceNames: [\"kms-luna-credentials\"]\n verbs: [\"get\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: secret-reader-binding\n namespace: custody-core\nsubjects:\n- kind: ServiceAccount\n name: harmonize-notary\n namespace: custody-core\nroleRef:\n kind: Role\n name: secret-reader\n apiGroup: rbac.authorization.k8s.io\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"secret-rotation","__idx":26},"children":["Secret rotation"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Automated Rotation with ESO"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# externalsecret.yaml\nspec:\n refreshInterval: 1h # Check Vault every hour\n\n # Optional: Force immediate sync on annotation change\n metadata:\n annotations:\n force-sync: \"2025-06-26T10:00:00Z\"\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Rotation Procedure"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# 1. Update secret in Vault\nvault kv put secret/app/luna-creds \\\n ca.pem=@ca-new.pem \\\n client.pem=@client-new.pem \\\n client.key=@client-new.key \\\n pin=\"new-pin\"\n\n# 2. Force ESO sync (optional, or wait for refreshInterval)\nkubectl annotate externalsecret external-kms-luna-secret \\\n force-sync=$(date +%s) \\\n -n custody-core\n\n# 3. Restart pods to pick up new secret\nkubectl rollout restart deployment/harmonize-notary -n custody-core\n\n# 4. Verify new secret is in use\nkubectl logs -n custody-core deployment/harmonize-notary | grep \"Connected to HSM\"\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"audit-logging","__idx":27},"children":["Audit logging"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Vault Audit Logging"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Enable audit logging\nvault audit enable file file_path=/vault/logs/audit.log\n\n# Query audit logs\ncat /vault/logs/audit.log | jq 'select(.request.path == \"secret/data/app/luna-creds\")'\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Kubernetes Audit Logging"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# audit-policy.yaml\napiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n- level: RequestResponse\n resources:\n - group: \"\"\n resources: [\"secrets\"]\n namespaces: [\"custody-core\"]\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"encryption-at-rest","__idx":28},"children":["Encryption at rest"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Vault Transit Engine"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Enable transit engine\nvault secrets enable transit\n\n# Create encryption key\nvault write -f transit/keys/custody-secrets\n\n# Encrypt secret before storing\nvault write transit/encrypt/custody-secrets \\\n plaintext=$(base64 <<< \"my-secret-value\")\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Kubernetes Encryption at Rest"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# encryption-config.yaml\napiVersion: apiserver.config.k8s.io/v1\nkind: EncryptionConfiguration\nresources:\n - resources:\n - secrets\n providers:\n - aescbc:\n keys:\n - name: key1\n secret: \n - identity: {}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"network-security","__idx":29},"children":["Network security"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["1. Vault TLS"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Configure Vault with TLS\nvault write pki/config/urls \\\n issuing_certificates=\"https://vault.example.com:8200/v1/pki/ca\" \\\n crl_distribution_points=\"https://vault.example.com:8200/v1/pki/crl\"\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["2. Network Policies"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"# network-policy.yaml\napiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: allow-vault-access\n namespace: custody-core\nspec:\n podSelector:\n matchLabels:\n app: harmonize-notary\n policyTypes:\n - Egress\n egress:\n - to:\n - podSelector:\n matchLabels:\n app: vault\n ports:\n - protocol: TCP\n port: 8200\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"troubleshooting","__idx":30},"children":["Troubleshooting"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"issue-eso-not-syncing-secrets","__idx":31},"children":["Issue: ESO not syncing secrets"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Symptoms"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["ExternalSecret shows ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["SecretSyncedError"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Kubernetes Secret not created or not updated"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Possible causes"]},":"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Vault authentication failed"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Vault path incorrect"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Vault policy doesn't allow read access"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Solution"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Check ExternalSecret status\nkubectl describe externalsecret external-kms-luna-secret -n custody-core\n\n# Check ESO logs\nkubectl logs -n external-secrets-system deployment/external-secrets\n\n# Test Vault authentication manually\nkubectl exec -it -n custody-core deployment/harmonize-notary -- \\\n vault login -method=kubernetes role=eso-reader\n\n# Verify Vault path\nvault kv get secret/app/luna-creds\n\n# Verify Vault policy\nvault policy read eso-reader\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"issue-vault-injector-secrets-not-injected","__idx":32},"children":["Issue: Vault Injector secrets not injected"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Symptoms"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["vault-agent sidecar not present"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/vault/secrets/"]}," directory empty"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Possible causes"]},":"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Annotations not set correctly."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Vault role not configured."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Service account not authorized."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Solution"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Check pod annotations\nkubectl get pod -n custody-core -o yaml | grep vault.hashicorp.com\n\n# Check vault-agent logs\nkubectl logs -n custody-core -c vault-agent\n\n# Verify Vault role\nvault read auth/kubernetes/role/eso-reader\n\n# Test authentication\nkubectl exec -it -n custody-core -c vault-agent -- \\\n cat /vault/secrets/ca.pem\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"issue-aws-secrets-manager-access-denied","__idx":33},"children":["Issue: AWS Secrets Manager access denied"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Symptoms"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Vault Deployer UI shows \"Access Denied\""]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Cannot retrieve secrets"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Possible causes"]},":"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["IAM role doesn't have permissions."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Secret doesn't exist."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Secret is in different region."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Solution"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Verify IAM role permissions\naws iam get-role-policy \\\n --role-name vault-deployer-role \\\n --policy-name SecretsManagerAccess\n\n# Verify secret exists\naws secretsmanager describe-secret \\\n --secret-id /ripple/custody///vault//config\n\n# Check region\naws secretsmanager list-secrets --region us-east-1 | grep ripple\n\n# Update IAM policy if needed\naws iam put-role-policy \\\n --role-name vault-deployer-role \\\n --policy-name SecretsManagerAccess \\\n --policy-document file://secrets-manager-policy.json\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"platform-specific-examples","__idx":34},"children":["Platform-specific examples"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"aws-cloudhsm-with-eso","__idx":35},"children":["AWS CloudHSM with ESO"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/deployment/integrate-kms/cloud-hsm/aws-cloudhsm"},"children":["AWS CloudHSM integration guide"]}," for complete example."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"thales-luna-hsm-with-eso","__idx":36},"children":["Thales Luna HSM with ESO"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/deployment/integrate-kms/on-premise-hsm/thales-luna#advanced-secret-management"},"children":["Thales Luna HSM integration guide - advanced secret management"]}," for complete example."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"blocksafe-hsm-with-eso","__idx":37},"children":["BlockSafe HSM with ESO"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/deployment/integrate-kms/on-premise-hsm/blocksafe#advanced-secret-management"},"children":["BlockSafe HSM integration guide - advanced secret management"]}," for complete example."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"mpc-with-eso","__idx":38},"children":["MPC with ESO"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/deployment/integrate-kms/mpc/overview"},"children":["MPC integration guide"]}," for complete example."]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"related-resources","__idx":39},"children":["Related resources"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://external-secrets.io/"},"children":["External Secrets Operator documentation"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.vaultproject.io/docs"},"children":["HashiCorp Vault documentation"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.aws.amazon.com/secretsmanager/"},"children":["AWS Secrets Manager documentation"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://kubernetes.io/docs/concepts/configuration/secret/"},"children":["Kubernetes Secrets documentation"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/custody/deployment/integrate-kms/overview"},"children":["Ripple Custody"," KMS integration guides"]}]}]},{"$$mdtype":"Tag","name":"hr","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"support","__idx":40},"children":["Support"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For assistance with advanced secret management:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["ESO issues"]},": ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/external-secrets/external-secrets"},"children":["External Secrets Operator GitHub"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Vault issues"]},": ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://support.hashicorp.com/"},"children":["HashiCorp Vault support"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["AWS issues"]},": ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://aws.amazon.com/support/"},"children":["AWS support"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Ripple Custody"," Issues"]},": Contact your Ripple liaison"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Required information for support requests"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Secret management tool (ESO, Vault Injector, AWS Secrets Manager)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Error messages from logs"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["ExternalSecret or pod annotations (sanitized)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Vault policy and role configuration (sanitized)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Kubernetes version and ESO version"]}]}]},"headings":[{"value":"Advanced secret management","id":"advanced-secret-management","depth":1},{"value":"Table of contents","id":"table-of-contents","depth":2},{"value":"Overview","id":"overview","depth":2},{"value":"What is advanced secret management?","id":"what-is-advanced-secret-management","depth":3},{"value":"Benefits","id":"benefits","depth":3},{"value":"External Secrets Operator (ESO)","id":"external-secrets-operator-eso","depth":2},{"value":"Overview","id":"overview-1","depth":3},{"value":"Prerequisites","id":"prerequisites","depth":3},{"value":"Step 1: Create SecretStore","id":"step-1-create-secretstore","depth":3},{"value":"Step 2: Create ExternalSecret","id":"step-2-create-externalsecret","depth":3},{"value":"Step 3: Reference secret in Helm values","id":"step-3-reference-secret-in-helm-values","depth":3},{"value":"Step 4: Verify secret rotation","id":"step-4-verify-secret-rotation","depth":3},{"value":"HashiCorp Vault Agent Injector","id":"hashicorp-vault-agent-injector","depth":2},{"value":"Overview","id":"overview-2","depth":3},{"value":"Prerequisites","id":"prerequisites-1","depth":3},{"value":"Step 1: Annotate pod for injection","id":"step-1-annotate-pod-for-injection","depth":3},{"value":"Step 2: Deploy and verify","id":"step-2-deploy-and-verify","depth":3},{"value":"AWS Secrets Manager","id":"aws-secrets-manager","depth":2},{"value":"Overview","id":"overview-3","depth":3},{"value":"Secrets stored","id":"secrets-stored","depth":3},{"value":"Security best practices","id":"security-best-practices","depth":3},{"value":"Accessing secrets","id":"accessing-secrets","depth":3},{"value":"Comparison: ESO vs Vault Injector vs AWS Secrets Manager","id":"comparison-eso-vs-vault-injector-vs-aws-secrets-manager","depth":2},{"value":"Recommendations","id":"recommendations","depth":3},{"value":"Security best practices","id":"security-best-practices-1","depth":2},{"value":"Principle of least privilege","id":"principle-of-least-privilege","depth":3},{"value":"Secret rotation","id":"secret-rotation","depth":3},{"value":"Audit logging","id":"audit-logging","depth":3},{"value":"Encryption at rest","id":"encryption-at-rest","depth":3},{"value":"Network security","id":"network-security","depth":3},{"value":"Troubleshooting","id":"troubleshooting","depth":2},{"value":"Issue: ESO not syncing secrets","id":"issue-eso-not-syncing-secrets","depth":3},{"value":"Issue: Vault Injector secrets not injected","id":"issue-vault-injector-secrets-not-injected","depth":3},{"value":"Issue: AWS Secrets Manager access denied","id":"issue-aws-secrets-manager-access-denied","depth":3},{"value":"Platform-specific examples","id":"platform-specific-examples","depth":2},{"value":"AWS CloudHSM with ESO","id":"aws-cloudhsm-with-eso","depth":3},{"value":"Thales Luna HSM with ESO","id":"thales-luna-hsm-with-eso","depth":3},{"value":"BlockSafe HSM with ESO","id":"blocksafe-hsm-with-eso","depth":3},{"value":"MPC with ESO","id":"mpc-with-eso","depth":3},{"value":"Related resources","id":"related-resources","depth":2},{"value":"Support","id":"support","depth":2}],"frontmatter":{"seo":{"title":"Advanced secret management"}},"lastModified":"2026-08-04T20:25:11.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/products/custody/deployment/integrate-kms/advanced-secret-management","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}