I have bean working some time with vault and to deploy it to our EKS cluster and then to get the secrets into our pods.
After many hours of searching i have found out that using kube-vault and vault-env. This gude uses tarraform to setup the resources you need in AWS.
Then deploy the kubevault with ui into to cluster that will use a s3 bucket and backend and autoseal it self during boot
And then we use the Banzia vault web-hook to when a ENV is detected to come from vault. It connects to vault and read the store it into the pod.
Resources
https://kubevault.com/docs/v0.3.0/guides/platforms/eks/
https://banzaicloud.com/blog/inject-secrets-into-pods-vault-revisited/
– We will have a vault running in our EKS cluster
– Secrets are stored in a S3 Bucket
– Vault keys are stored in AWS
– During startup vault will unseal it self
– When a pod starts the secrets from vault will be exposed ans a ENV settings for the pod (12 faktor style)
Lets start with AWS and setup the resources you need. This will only create the resources for vault and you need to have your other EKS resources already done.
Im using terraform so this is a terraform manifest for running.
#
# Create as3 bucket for the vaul secrets
#
resource "aws_s3_bucket" "vault-vault-bucket" {
bucket = "vault-gdtegd"
acl = "private"
tags = {
Name = "vault-gdtegd"
Environment = "prod"
DONT_DELETE = "true"
}
}
#
# Policy for vaul service account s3 bucket
#
resource "aws_iam_policy" "vault_s3" {
name = "vaults3"
path = "/"
description = "vault s3 access"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VaultListBuckets",
"Effect": "Allow",
"Action": [
"s3:ListAllMyBuckets",
"s3:HeadBucket"
],
"Resource": "*"
},
{
"Sid": "VaultAccessBuckets",
"Effect": "Allow",
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::${aws_s3_bucket.vault-vault-bucket.id}",
"arn:aws:s3:::${aws_s3_bucket.vault-vault-bucket.id}/*"
]
}
]
}
EOF
}
#
# Create aws KMS key
#
resource "aws_kms_key" "vault" {
description = "vault_key"
}
#
# Policy for vaul service account for the KMS
#
data "aws_caller_identity" "current" {}
data "aws_region" "current" {}
resource "aws_iam_policy" "vault_kms" {
name = "vaultkms"
path = "/"
description = "vault kms access"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VaultUnsealerEncryptDecryptKms",
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:Encrypt",
"kms:DescribeKey"
],
"Resource": "arn:aws:kms:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:key/${aws_kms_key.vault.key_id}"
},
{
"Sid": "VaultUnsealerGetKMS",
"Effect": "Allow",
"Action": "kms:ListKeys",
"Resource": "*"
}
]
}
EOF
}
#
# Policy for vaul service account for the SSM
#
resource "aws_iam_policy" "vault_ssm" {
name = "vaultssm"
path = "/"
description = "vault ssm access"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VaultUnsealerParametersAccess",
"Effect": "Allow",
"Action": [
"ssm:PutParameter",
"ssm:DeleteParameter",
"ssm:GetParameters"
],
"Resource": "arn:aws:ssm:*:*:parameter/*"
}
]
}
EOF
}
#
# Create user
#
resource "aws_iam_user" "vault-k8s" {
name = "vault-k8s"
path = "/system/"
tags = {
used_in = "k8s"
}
}
resource "aws_iam_access_key" "vault-k8s" {
user = aws_iam_user.vault-k8s.name
}
#
# Create role for vault
#
resource "aws_iam_role" "vault-k8s" {
name = "vault-k8s"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Federated": "arn:aws:iam::{ YOUR VAULT USER ID }:oidc-provider/oidc.eks.eu-north-1.amazonaws.com/id/{ YOUR CLUSTER ID }"
},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": {
"StringEquals": {
"oidc.eks.eu-north-1.amazonaws.com/id/{ YOUR CLUSTER ID } :sub": "system:serviceaccount:vault-k8s:vault-k8s"
}
}
}
]
}
EOF
tags = {
inused = "k8s"
}
}
#
# Applied policy to user and role
#
resource "aws_iam_role_policy_attachment" "vault_s3" {
role = aws_iam_role.vault-k8s.name
policy_arn = aws_iam_policy.vault_s3.arn
}
resource "aws_iam_role_policy_attachment" "vault_kms" {
role = aws_iam_role.vault-k8s.name
policy_arn = aws_iam_policy.vault_kms.arn
}
resource "aws_iam_role_policy_attachment" "vault_ssm" {
role = aws_iam_role.vault-k8s.name
policy_arn = aws_iam_policy.vault_ssm.arn
}
resource "aws_iam_policy_attachment" "vault_s3_u" {
name = "vault_s3_u"
users = ["${aws_iam_user.vault-k8s.name}"]
policy_arn = aws_iam_policy.vault_s3.arn
}
resource "aws_iam_policy_attachment" "vault_kms_u" {
name = "vault_kms_u"
users = ["${aws_iam_user.vault-k8s.name}"]
policy_arn = aws_iam_policy.vault_kms.arn
}
resource "aws_iam_policy_attachment" "vault_ssm_u" {
name = "vault_ssm_u"
users = ["${aws_iam_user.vault-k8s.name}"]
policy_arn = aws_iam_policy.vault_ssm.arn
}
#
#
# Output
#
output "vault-kms-key" {
value = aws_kms_key.vault.key_id
}
output "secret_vault-k8s_id" {
value = aws_iam_access_key.vault-k8s.id
}
output "secret_vault-k8s_key" {
value = aws_iam_access_key.vault-k8s.secret
}I have a small bash script that setup and install the vault operator. With this i can run in in my pipeline and use it to upgrade as well.
#!/bin/bash
# Setup repos
helm repo add appscode https://charts.appscode.com/stable/
helm repo update
# Install the vault operator
helm upgrade --install vault-operator appscode/vault-operator --version v0.3.0 --namespace kube-system
#Install the vault catalog
helm upgrade --install vault-catalog appscode/vault-catalog --version v0.3.0 --namespace kube-system
We need to add an update so we can use the latest vault that today is (1.5.4)
create the file vaultserver-1.5.4.yaml
apiVersion: catalog.kubevault.com/v1alpha1
kind: VaultServerVersion
metadata:
name: 1.5.4
spec:
vault:
image: vault:1.5.4
exporter:
image: kubevault/vault-exporter-linux-amd64:v0.3.0
unsealer:
image: kubevault/vault-unsealer:v0.3.0
version: 1.5.4kubectl apply -f vaultserver-1.5.4.yamlTo verify they are installed run this command
kubectl get crds -n kube-systemYou should see some vault names there
First we need some settings
– The bucket you created with terraform
– KeyID and secret for your vault user created with terraform
– KMS key
– Vault UI
All this should be printed when you run terraform apply.
Now lest move the secrets into base64
echo -n "YOUR AWS SECRET / KEYID" | base64
Create you deploy YAML
apiVersion: v1
kind: ConfigMap
metadata:
name: vault-ui
namespace: vault
data:
vault.hcl: |
ui = true
---
---
apiVersion: v1
kind: Namespace
metadata:
name: vault
---
apiVersion: v1
kind: Secret
metadata:
name: vault-aws-secrets
namespace: vault
type: Opaque
data:
access_key: {AWS KEY ID }
secret_key: {AWS SECRET}
---
apiVersion: kubevault.com/v1alpha1
kind: VaultServer
metadata:
name: vault
namespace: vault
spec:
configSource:
configMap:
name: vault-ui
replicas: 1
version: "1.5.4"
backend:
s3:
bucket: "{BUCKET NAME}"
region: "eu-north-1"
credentialSecret: vault-aws-secrets
unsealer:
secretShares: 4
secretThreshold: 2
mode:
awsKmsSsm:
region: "eu-north-1"
kmsKeyID: "{ KMS KEY ID }"
credentialSecret: vault-aws-secrets
---
apiVersion: v1
kind: Service
metadata:
name: vault-ingress
namespace: vault
spec:
selector:
app: vault
cluster: vault
type: ClusterIP
ports:
- port: 443
name: https
targetPort: 8200
---
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: "vault"
namespace: vault
annotations:
kubernetes.io/ingress.class: "traefik"
cert-manager.io/cluster-issuer: "letsencrypt"
kubernetes.io/tls-acme: "true"
spec:
tls:
- hosts:
- vault.
secretName: vaultapps-tls
rules:
- host: vault.
paths:
- path: /
backend:
serviceName: vault-ingress
servicePort: 443
---
Now we should have a running vault that are unseal. Lets get the root token and connect to it.
Here is a script to get the root token save it as get_vault_token.sh
#!/bin/bash
#
# This will get and print out the vault root token
# the token can then be used to make changes to vault as a admin.
#
# BEFORE YOU CAN RUN THIS GIVE TERRAFORM AWS USER ADMIN RIGHTS ADD IT MANUALL THEN TERRAFORM WILL REMOVE FOR YOU
#
aws ssm get-parameter --name vault-root-token --region eu-north-1 --output json | jq -r '.Parameter.Value' | base64 -d - > root.enc
export VAULT_TOKEN=$(aws kms decrypt --ciphertext-blob fileb://root.enc --output text --query Plaintext --encryption-context "Tool=vault-unsealer" --region eu-north-1 | base64 -d -)
rm root.enc
export VAULT_ADDR='https://127.0.0.1:8200'
To activate it run . ./get_vault_token.sh You did see the first . <- important
Now start a kube proxy to the vault service create a file start_vault_proxy.sh
#!/bin/bash
#
# This will setup a vault proxy so localhost:8200 can proxy to vault
# Fix fix tls ..
kubectl port-forward service/vault 8200:8200 -n vault &
echo "Need to sleep 5 to make the proxy connect"
sleep 5./start_vault_proxy.sh to start up the proxy it will jump into the background
Now we can connect to vault
vault status -tls-skip-verifyWe need to have the -tls-skip-verify when we connect throw the proxy and dont have the ca cert of vault.
Now its time to get some values into vault and to connect our pods with with vault.
In this we are to make the following.
– Let vault connect to k8s to create token
– Create a secret in vault
– Create a vault policy that grant access to our secret
– Create a role that are restricted to the policy and to the namespace
Allow vault to make new k8s token Verify that you have the correct service account and namespace for vault. Check in you deployments of vault.
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: role-tokenreview-binding
namespace: default
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: vault
namespace: vaultSounds complicated well here is a script i run when setup helm charts.
It set up what you need and also takes the values.yaml file from the helm chart and add it to vault.
It also add a lock secrets so it sill not overrite the secret with every deploy.
#
# This script will setup a vault to use with a namespace.
# It will to the following
#- setup vault secret in the correct path
#- setup a k8s policy for service account in namespaces to access that path
#- if not setup fill secrets with default values
#
#
#After this script is run we can now use vaul-hook to get secrets into our pod as env
D_DEPLOYNAME="pipeline"
#Check so that we have a deployname
if [[ -n "${DEPLOYNAME}" ]]; then
DEPLOY=$(echo $DEPLOYNAME | sed 's/\(.*\)/\L\1/' | sed 's/[\/.]/-/g' )
#Cluster env is not set defult to standrad
echo "Set DEPLOYNAME to ${DEPLOY}"
D_DEPLOYNAME="${DEPLOY}"
fi
#Set defulat
D_HELMCHART=$D_DEPLOYNAME
D_NAMESPACE=$D_DEPLOYNAME
#If we have a namespace env
if [[ -n "${NAMESPACE}" ]]; then
#Cluster env is not set defult to standrad
echo "Set NAMESPACE to ${NAMESPACE}"
D_NAMESPACE=${NAMESPACE}
fi
#If we have a helmchart env
if [[ -n "${HELMCHART}" ]]; then
#Cluster env is not set defult to standrad
echo "Set HELMCHART to ${HELMCHART}"
D_HELMCHART=${HELMCHART}
fi
echo "Have we setup this namespace before ?"
VAULT_SETUP_TEST=$(vault kv get -tls-skip-verify /kv/k8s/$D_NAMESPACE-lock/locked)
echo $VAULT_SETUP_TEST
echo ${VAULT_SETUP_TEST:0:2}
if [ "${VAULT_SETUP_TEST:0:2}" == "==" ]; then
echo "Vault already setup"
else
echo "No vault setup lets add it"
echo "Lets add policy for namespace that gives read access"
echo 'path "kv/k8s/'$D_NAMESPACE'" { capabilities = ["read"]}' | vault policy write -tls-skip-verify $D_NAMESPACE -
echo "Create a role that grant access for the default user in the namespace"
vault write -tls-skip-verify auth/kubernetes/role/vault-$D_NAMESPACE bound_service_account_names=default bound_service_account_namespaces=$D_NAMESPACE policies=$D_NAMESPACE ttl=24h
echo "Write the default values form a helm chart into vault"
VALUESFILE="/code/helm/$D_HELMCHART/values.yaml"
echo "Values file URL $VALUESFILE"
if [ -f $VALUESFILE ]; then
echo "Adding the values file data"
cat $VALUESFILE | yq . | vault kv put -tls-skip-verify /kv/k8s/$D_NAMESPACE -
else
echo "No values file foundls"
fi
echo "Added tha data lets lock it "
vault kv put -tls-skip-verify /kv/k8s/$D_NAMESPACE-lock/locked dontmove="OK"
fi
echo "Renew token"
vault token renew -tls-skip-verify >> /dev/null
echo 'path "kv/k8s/'$D_NAMESPACE'" { capabilities = ["read"]}' | vault policy write -tls-skip-verify $D_NAMESPACE -
Here we add a vault policy that gived the accees read to the path /kv/k8s/$NAMESPACE
So if you namespace /pipline here then it will give read access to /kv/k8s/pipline
ault write -tls-skip-verify auth/kubernetes/role/vault-$D_NAMESPACE bound_service_account_names=default bound_service_account_namespaces=$D_NAMESPACE policies=$D_NAMESPACE ttl=24h
Now we are creating a role that have the policy added before and the role is locked in k8s. so it must be the service account of defalt (Stanard when you deploy) and it need to be in the namespace we set before example “pipline”
cat $VALUESFILE | yq . | vault kv put -tls-skip-verify /kv/k8s/$D_NAMESPACE -This is a small hack that take the values.yaml from my helm charts and konvert them into json and upload them as secrets.
So I get some data in vault and it gets easy to find where you put you data.
Now we need to install the vault-hook. It detects that we want some secrets from vault and injects a script that will connect to vault. Get the tokens and add the secrets as ENV in your pod.
You need to have the banziacloud helm repo added
kubectl create namespace vault-hook
helm upgrade --namespace vault-hook --install vault-secrets-webhook banzaicloud-stable/vault-secrets-webhook --wait
Lets create a test pod
apiVersion: apps/v1
kind: Deployment
metadata:
name: hello-secrets
spec:
replicas: 1
selector:
matchLabels:
app: hello-secrets
template:
metadata:
labels:
app: hello-secrets
annotations:
vault.security.banzaicloud.io/vault-addr: "https://vault.vault.svc:8200"
vault.security.banzaicloud.io/vault-role: "vault-slackbot" #
vault.security.banzaicloud.io/vault-skip-verify: "true"
spec:
serviceAccountName: default
containers:
- name: alpine
image: alpine
command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
env:
- name: AWS_SECRET_ACCESS_KEY
value: "vault:kv/k8s/slackbot#auth"Here we have a pod that are called slackbot so what we need before this will work
– Have a policy grating access to /kv/k8s/slackbot
– Have a role that have access from service user default and namespace slackbot, And are using the policy
– A secrets in /kv/k8s/slackbot
root@20927fd8e1e1:/code/setup# vault kv get -tls-skip-verify /kv/k8s/slackbot
Handling connection for 8200
======== Data ========
Key Value
--- -----
auth 2hsfsgffehgggs
cluster int
deployname slackbot
gitsha latest
replicaCount 1
apply the tets pod in the slackbot namespace
kubectl apply -f test_vault.yaml -n slackbotLook in the logs to verify things are working
root@20927fd8e1e1:/code/k8s/pre/vault/test# kubectl logs -f hello-secrets-7944dfcf86-rkrk6 -n slackbot
time="2020-10-29T07:09:30Z" level=info msg="received new Vault token" app=vault-env
time="2020-10-29T07:09:30Z" level=info msg="initial Vault token arrived" app=vault-env
time="2020-10-29T07:09:30Z" level=info msg="spawning process: [sh -c echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000]" app=vault-env
2hsfsgffehgggs
going to sleep...