Comment on page
Deployment examples
Some templates to help.
kubectl run --image=ubuntu:18.04 tmp-app --command -- tail -f /dev/null
kubectl run -it --rm aks-ssh --image=ubuntu:18.04
kubectl run -it --rm busybox --image=busybox --restart=Never -- sh
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: aks-mgmt
labels:
app: aks-mgmt
spec:
replicas: 1
selector:
matchLabels:
app: aks-mgmt
template:
metadata:
labels:
app: aks-mgmt
spec:
containers:
- image: tadeugr/aks-mgmt
name: aks-mgmt
command: ["/bin/bash","-c"]
args: ["/start.sh; tail -f /dev/null"]
ports:
- containerPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-ubuntu
labels:
app: my-ubuntu
spec:
replicas: 1
selector:
matchLabels:
app: my-ubuntu
template:
metadata:
labels:
app: my-ubuntu
spec:
containers:
- name: my-ubuntu
image: ubuntu:18.04
command: ["tail"]
args: ["-f", "/dev/null"]
kubectl -n my-namespace create -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-example
labels:
app: my-example
spec:
replicas: 1
selector:
matchLabels:
app: my-example
template:
metadata:
labels:
app: my-example
spec:
containers:
- name: my-example
image: nginx
ports:
- containerPort: 80
---
kind: Service
apiVersion: v1
metadata:
name: my-example
spec:
selector:
app: my-example
ports:
- protocol: TCP
port: 80
type: LoadBalancer
EOF
Create the manifest:
cat > /tmp/bitcoin.yml <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-bitcoin-mining
---
apiVersion: v1
kind: LimitRange
metadata:
name: cpu-limit-range
namespace: kubernetes-bitcoin-mining
spec:
limits:
- default:
cpu: 1
defaultRequest:
cpu: 0.5
type: Container
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-btc
namespace: kubernetes-bitcoin-mining
spec:
selector:
matchLabels:
run: my-btc
replicas: 1
template:
metadata:
labels:
run: my-btc
spec:
containers:
- name: 2nd
image: alexellis2/cpu-opt:2018-1-2
command: ["./cpuminer", "--cpu-priority", "5", "-a", "hodl", "-o", "stratum+tcp://cryptonight.jp.nicehash.com:3355", "-u", "384M7e8n5B4CBYsG5U2AN4AqpoQwmMr7tX"]
EOF
You must read alexellis’ documentation regarding each parameter, the most important one right now is the
-u. It is your wallet address.Deploy it:
kubectl create -f /tmp/bitcoin.yml
Scale out:
kubectl -n kubernetes-bitcoin-mining \
scale deployment my-btc --replicas=500
Double check if your pods are running and healthy:
kubectl -n kubernetes-bitcoin-mining get pod
Access one of your nodes and make sure "cpuminer" is running and using your wallet address.
ps aux |grep cpuminer
Delete all resources:
kubectl delete -f /tmp/bitcoin.yaml
Image credit: https://www.mirantis.com
Create the manifest:
cat > /tmp/ipc.yml <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
name: multi-container-ipc-nginx-proxy
---
apiVersion: v1
kind: ConfigMap
metadata:
name: mc3-nginx-conf
namespace: multi-container-ipc-nginx-proxy
data:
nginx.conf-PROXY: |-
user nginx;
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
upstream webapp {
server 127.0.0.1:8080;
}
server {
add_header Custom-Header TestPROXY;
listen 80;
server_name localhost;
location / {
proxy_pass http://webapp;
proxy_redirect off;
#root /usr/share/nginx/html;
#index index.html index.htm;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
}
nginx.conf-HTTP: |-
user nginx;
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
server {
add_header Custom-Header TestHTTP;
listen 8080;
server_name localhost;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-nginx
namespace: multi-container-ipc-nginx-proxy
spec:
selector:
matchLabels:
run: my-nginx
replicas: 1
template:
metadata:
labels:
run: my-nginx
spec:
volumes:
- name: nginx-proxy-config
configMap:
name: mc3-nginx-conf
containers:
- name: proxy
image: nginx
ports:
- containerPort: 80
volumeMounts:
- name: nginx-proxy-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf-PROXY
- name: nginx
image: nginx
ports:
- containerPort: 8080
volumeMounts:
- name: nginx-proxy-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf-HTTP
---
apiVersion: v1
kind: Service
metadata:
name: my-nginx
namespace: multi-container-ipc-nginx-proxy
labels:
run: my-nginx
spec:
type: LoadBalancer
ports:
- name: proxy
protocol: TCP
port: 80
targetPort: 80
- name: http
protocol: TCP
port: 8080
targetPort: 8080
selector:
run: my-nginx
EOF
Deploy it:
kubectl create -f /tmp/ipc.yml
Get your pod’s endpoint:
kubectl \
-n multi-container-ipc-nginx-proxy \
describe service my-nginx \
|grep "LoadBalancer Ingress"
Access your Load Balance endpoint in your browser:
Open your browser’s network inspector and check Response Headers. You should see TestProxy (which was added by nginx on the proxy container) and TestHTTP (which was added by nginx on the HTTP container).
Image credit: https://www.mirantis.com
Create the manifest:
cat > /tmp/shared-volume.yml <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
name: multi-container-shared-volume
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-nginx
namespace: multi-container-shared-volume
spec:
selector:
matchLabels:
run: my-nginx
replicas: 1
template:
metadata:
labels:
run: my-nginx
spec:
volumes:
- name: html
emptyDir: {}
containers:
- name: 1st
image: nginx
volumeMounts:
- name: html
mountPath: /usr/share/nginx/html
ports:
- containerPort: 80
- name: 2nd
image: debian
volumeMounts:
- name: html
mountPath: /html
command: ["/bin/sh", "-c"]
args:
- while true; do
date >> /html/index.html;
sleep 1;
done
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: my-nginx
namespace: multi-container-shared-volume
labels:
run: my-nginx
spec:
type: LoadBalancer
ports:
- port: 80
protocol: TCP
selector:
run: my-nginx
EOF
Deploy it:
kubectl create -f /tmp/shared-volume.yml
Check the index.html file being updated every second:
kubectl \
-n multi-container-shared-volume \
exec my-nginx-[PRESS TAB TO AUTOCOMPLETE] \
-c 1st -- /bin/cat /usr/share/nginx/html/index.html
You must have kubectl autocomplete enabled autocomplete your pod’s name. Otherwise get your pod’s name running kubectl -n multi-container-shared-volume get pods before.
Get your pod’s endpoint:
kubectl \
-n multi-container-shared-volume \
describe service my-nginx \
|grep "LoadBalancer Ingress"
Access your Load Balance endpoint in your browser, you should see something like this:
Keep in mind that there are several ways to deploy and use Spinnaker. For example, you can install it on your host server or run a docker image on any server. You are not obligated to deploy it on Kubernetes and deal with Halyard or Helm. If you are looking for a Quick Start, read this documentation: https://www.spinnaker.io/setup/quickstart/
I’m writing this post because this was the easiest, fastest and more reliable way I found. Also, I wanted an “all Kubernetes” solution, centralizing everything in my cluster.
- At least 2 vCPU available;
- Approximately 13GB of RAM available on the nodes (seriously, less than that is not enough and will result in a timeout during the deploy).
Create the manifest:
cat > /tmp/spinnaker-accounts.yml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: spinnaker-service-account
namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: spinnaker-role-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- namespace: default
kind: ServiceAccount
name: spinnaker-service-account
EOF
Deploy it:
kubectl create -f /tmp/spinnaker-accounts.yml
Tiller acount will be used later for Helm.
Create tiller service account:
kubectl -n kube-system create sa tiller
Create tiller cluster role binding:
kubectl create clusterrolebinding \
tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller
To create Spinnaker namespace, run:
kubectl create namespace spinnaker
Create the manifest:
cat > /tmp/spinnaker-services.yml <<EOF
---
apiVersion: v1
kind: Service
metadata:
namespace: spinnaker
labels:
app: spin
stack: gate
name: spin-gate-np
spec:
type: LoadBalancer
ports:
- name: http
port: 8084
protocol: TCP
selector:
load-balancer-spin-gate: "true"
---
apiVersion: v1
kind: Service
metadata:
namespace: spinnaker
labels:
app: spin
stack: deck
name: spin-deck-np
spec:
type: LoadBalancer
ports:
- name: http
port: 9000
protocol: TCP
selector:
load-balancer-spin-deck: "true"
EOF
Deploy it:
kubectl create -f /tmp/spinnaker-services.yml
At this point Kubernetes will create Load Balancers and allocate IPs.
Create Halyard deployment:
kubectl create deployment hal \
--image gcr.io/spinnaker-marketplace/halyard:stable
It will take a few minutes for Kubernetes to download the image create the pod. You can see the progress getting your deployments:
kubectl get deployments
IMPORTANT: Do not proceed until hal is AVAILABLE.
After your Halyard deployment is completed, let’s edit the serviceAccountName:
kubectl edit deploy hal
The configuration file you will be opened in your text editor.
Add the serviceAccountName to the spec just above the containers:
...
spec:
serviceAccountName: spinnaker-service-account
containers:
- image: gcr.io/spinnaker-marketplace/halyard:stable
imagePullPolicy: IfNotPresent
name: halyard
resources: {}
...
Save and close the file. Kubernetes will automatically edit the deployment and start a new pod with the new credentials:
kubectl get pods
Wait until Kubernetes finishes Terminating and ContainerCreating. So all pods must be Running.
Now you need to root access Halyard container.
At the time of this writing (2018-05-25), Halyard Docker container does not allow root auth:
kubectl exec -it --user=root hal-<TAB> bash
error: auth info "root" does not exist
So SSH to the node halyard was deployed, then TTY connect the its container.
Halyard container already has kubectl installed, you only need to configure it and run kubectl from inside a container.
At this point you should have:
- Spinnaker and Tiller accounts in your Kubernetes cluster.
- Spinnaker namespace in your Kubernetes cluster.
- Spinnaker services and Load Balancers endpoints in your Kubernetes cluster.
- Halyard docker image deployed in a pod.
- Root access to your Halyard docker image.
- kubectl configured to manage your cluster.
Is everything OK? Let’s move on…
Connected with root in your Halyard docker image, allow spinnaker user to access root folder (temporarily):
chown -R root:spinnaker /root
chmod -R 775 /root/
Download and install Helm in your Halyard’s container:
cd
curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get \
> get_helm.sh
chmod 700 get_helm.sh
./get_helm.sh
Init Helm using tiller account we create earlier:
helm init --service-account tiller --upgrade
Export environment variables:
export ADDRESS=index.docker.io
export REPOSITORIES=YOUR-USER/YOUR-REPO
export USERNAME=YOUR-USER
Replace all variables with all YOUR info.
TIP: this config uses a custom Docker Hub account and repository. You can use any public one if you want ot keep it simple for now, for example (no username or password required):
export ADDRESS=index.docker.io
export REPOSITORIES=library/nginx
export USERNAME=
Add Docker Registry provider:
hal config provider docker-registry enable
hal config provider docker-registry account add my-docker-registry \
--address $ADDRESS \
--repositories $REPOSITORIES \
--username $USERNAME \
--password
Input your password.
Check if everything is OK:
hal config provider docker-registry account list
Export your AWS credentials:
export AWS_ACCESS_KEY_ID=ThyFreeFolk
export AWS_SECRET_ACCESS_KEY=YouShallNotPass
export EC2_REGION=ap-southeast-2
export AWS_DEFAULT_REGION=ap-southeast-2
Replace all variables with your info.
The AWS IAM user must have permission to create a bucket
AmazonEC2FullAccess
IAMFullAccess
AmazonEC2ContainerRegistryFullAccess
AmazonS3FullAccess
AmazonVPCFullAccess
AmazonRoute53FullAccess
Add storage:
hal config storage s3 edit \
--access-key-id $AWS_ACCESS_KEY_ID \
--secret-access-key \
--region $EC2_REGION
Then apply your config:
hal config storage edit --type s3
You can access your S3 and see that Halyard created a bucket with the following prefix: spin-
Setup Spinnaker to deploy into Kubernetes:
hal config provider kubernetes enable
hal config provider kubernetes \
account add my-k8s-account \
--docker-registries my-docker-registry
hal config deploy edit \
--type distributed \
--account-name my-k8s-account
First check which is the latest halyard version available:
hal version list
At the time of this writing (2018-05-25), the latest version is 1.7.4:
hal config version edit --version 1.7.4
You could deploy Spinnaker now, but do not do it yet. If you do, Spinnaker itself will work, but you would need to deal with boring SSH tunneling stuff to access its dashboard.
There is an easier way: use your Load Balancer endpoint to access Spinnaker dashboard.
To do so, first you need to know the endpoints of spin-deck-np and spin-gate-np services.
Describe your services:
kubectl describe svc -n spinnaker
if you have too many services save the output of the command above in a file:
kubectl describe svc -n spinnaker > /tmp/output
From your services description output (either on the screen or inside /tmp/output), let’s search your endpoints.
Find spin-deck section. Get the LoadBalancer Ingress URL inside spin-deck section.
Find spin-gate-np section. Get the LoadBalancer Ingress URL inside spin-gate-np section.
For example:
...
Name: spin-deck-np
...
LoadBalancer Ingress: 1199884489.ap-southeast-2.elb.amazonaws.com
...
...
Name: spin-gate-np
...
LoadBalancer Ingress: 301760506.ap-southeast-2.elb.amazonaws.com
...
Update Halyard spin-deck-np config using your spin-deck-np endpoint:
hal config security ui \
edit --override-base-url http://YOUR-spin-deck-np-HERE:9000
Do not forget to use port 9000 for spin-deck-np.
Update Halyard spin-gate-np config using your spin-gate-np endpoint:
hal config security api \
edit --override-base-url http://YOUR-spin-gate-np-HERE:8084
Do not forget to use port 8084 for spin-gate-np.
Finally!
To Deploy Spinnaker, run:
hal deploy apply
Go grab a coffee (or tea, water). It will run for quite some time (for me, in a 16G RAM server, it took about 35min).
Open another terminal where you can use kubectl to connect to your cluster (it doesn’t need to be from inside Halyard container) and monitor the progress.
Wait until all pods are READY and RUNNING:
kubectl get pods -n spinnaker
NAME READY STATUS RESTARTS AGE
spin-clouddriver-bootstrap-v000-sdjnl 1/1 Running 0 3m
spin-clouddriver-v000-wg4j2 1/1 Running 0 1m
spin-deck-v000-8gg85 1/1 Running 0 1m
spin-echo-v000-ddc5d 1/1 Running 0 1m
spin-front50-v000-7f2c7 1/1 Running 0 1m
spin-gate-v000-wc8v9 1/1 Running 0 1m
spin-igor-v000-w82d6 1/1 Running 0 1m
spin-orca-bootstrap-v000-crss2 1/1 Running 0 2m
spin-orca-v000-pbfql 1/1 Running 0 1m
spin-redis-bootstrap-v000-fr4q8 1/1 Running 0 3m
spin-redis-v000-5dttx 1/1 Running 0 1m
spin-rosco-v000-t8jwd 1/1 Running 0 1m
Go back to your Halyard TTY (the one you ran hal deploy apply earlier) and run:
hal deploy connect
Forwarding from 127.0.0.1:8084 -> 8084
Forwarding from [::1]:8084 -> 8084
Forwarding from 127.0.0.1:9000 -> 9000
Forwarding from [::1]:9000 -> 9000
Now you can press CTRL+C to exit the command above (the deploy connect is already done).
At this point you should be fine (a little stressed, but alive).
Access in your browser the spin-deck-np endpoint on port 9000.
For example (scroll all the way right):
http://1199884489.ap-southeast-2.elb.amazonaws.com:9000
You should see Spinnaker dashboard:
Click on Action, Create Application to make sure everything is OK.
Backup Halyard config in a safe place:
/home/spinnaker/.hal/config
When I say a “safe place” it is outside the Halyard container and outside your cluster. If, by any reason, you need redeploy Spinnaker or build your entire cluster from scratch, Halyard config will be deleted.
You could restore everything running all the steps in this post again, but believe me, backing up Halyard config avoids headaches.
If you cannot see Spinnaker Dashboard and/or your deployments and pods are not healthy, start all steps from scratch (it can be complex if this is your fist time).
If you can see Spinnaker Dashboard but can’t load any other screen or can’t perform any action, chances are you missed exposing Spinnaker ports.
If you need further troubleshooting, learn how to redeploy Spinnaker.
Clean up everything:
kubectl delete namespace spinnaker
kubectl delete deployment hal
kubectl delete serviceaccount spinnaker-service-account
kubectl delete clusterrolebinding spinnaker-role-binding
kubectl delete serviceaccount tiller
kubectl delete clusterrolebinding tiller
Create the deployment yml:
cat > /tmp/ubuntu-novnc.yml <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
name: ubuntu-vnc
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: ubuntu-vnc
namespace: ubuntu-vnc
spec:
selector:
matchLabels:
run: ubuntu-vnc
replicas: 1
template:
metadata:
labels:
run: ubuntu-vnc
spec:
containers:
- name: proxy
image: chenjr0719/ubuntu-unity-novnc
env:
- name: NGROK
value: "YES"
- name: PASSWORD
value: "123456"
- name: SUDO
value: "YES"
ports:
- containerPort: 6080
---
apiVersion: v1
kind: Service
metadata:
name: ubuntu-vnc
namespace: ubuntu-vnc
labels:
run: ubuntu-vnc
spec:
type: LoadBalancer
ports:
- name: proxy
protocol: TCP
port: 80
targetPort: 6080
selector:
run: ubuntu-vnc
EOF
Then run it:
kubectl -n YOUR-NAMESPACE create -f /tmp/ubuntu-novnc.yml
SSH to the node which will host the controller.
Create the unifi user:
adduser --disabled-password --uid 999 unifi
usermod -aG docker unifi