marko/docker-development-youtube-series
1
0
Fork 0
mirror of https://github.com/marcel-dempers/docker-development-youtube-series.git synced 2025-06-06 17:01:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

add init containers to add dummy data

Browse Source
This commit is contained in:
marcel-dempers 2020-10-04 21:21:24 +11:00 committed by Marcel Dempers
parent 507ca6d65f
commit a4a663fb96
14 changed files with 353 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
kubernetes/servicemesh/applications/playlists-api/app.go
View File

@ -10,14 +10,12 @@ import (
"bytes"
"io/ioutil"
"context"
"time"
"github.com/go-redis/redis/v8"
)
var environment = os.Getenv("ENVIRONMENT")
var redis_host = os.Getenv("REDIS_HOST")
var redis_port = os.Getenv("REDIS_PORT")
var ctx = context.Background()
var rdb *redis.Client
@ -88,45 +86,6 @@ func main() {
})
rdb = r
//init dummy data into redis
duration, _ := time.ParseDuration("4s")
dummyData := `
[
{
"id" : "1",
"name" : "CI/CD",
"videos": [ { "id" : "OFgziggbCOg"}, { "id" : "myCcJJ_Fk10"}, { "id" : "2WSJF7d8dUg"}]
},
{
"id" : "2",
"name" : "K8s in the Cloud",
"videos": [ { "id" : "QThadS3Soig"}, { "id" : "eyvLwK5C2dw"}]
},
{
"id" : "3",
"name" : "Storage and MessageBrokers",
"videos": [ { "id" : "JmCn7k0PlV4"}, { "id" : "_lpDfMkxccc"}]
},
{
"id" : "4",
"name" : "K8s Autoscaling",
"videos": [ { "id" : "jM36M39MA3I"}, { "id" : "FfDI08sgrYY"}]
}
]
`
retry(10, duration, func() (err error){
err = rdb.Set(ctx, "playlists",dummyData , 0).Err()
if err != nil {
fmt.Println("error occured connecting to Redis, retrying...")
return err
}
fmt.Println("Redis dummy data initialised")
return nil
})
fmt.Println("Running...")
log.Fatal(http.ListenAndServe(":10010", router))
}
@ -135,6 +94,7 @@ func getPlaylists()(response string){
playlistData, err := rdb.Get(ctx, "playlists").Result()
if err != nil {
fmt.Println(err)
fmt.Println("error occured retrieving playlists from Redis")
return "[]"
}
@ -161,22 +121,6 @@ type stop struct {
error
}
func retry(attempts int, sleep time.Duration, fn func() error) error {
if err := fn(); err != nil {
if s, ok := err.(stop); ok {
// Return the original error for later checking
return s.error
}
if attempts--; attempts > 0 {
time.Sleep(sleep)
return retry(attempts, 2*sleep, fn)
}
return err
}
return nil
}
func cors(writer http.ResponseWriter) () {
if(environment == "DEBUG"){
writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")

38
kubernetes/servicemesh/applications/playlists-api/deploy.yaml
View File

@ -49,44 +49,6 @@ spec:
port: 80
targetPort: 10010
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: playlists-db
labels:
app: playlists-db
spec:
selector:
matchLabels:
app: playlists-db
replicas: 1
template:
metadata:
labels:
app: playlists-db
spec:
containers:
- name: playlists-api
image: redis:6.0-alpine
ports:
- containerPort: 6379
---
apiVersion: v1
kind: Service
metadata:
name: playlists-db
labels:
app: playlists-db
spec:
type: ClusterIP
selector:
app: playlists-db
ports:
- protocol: TCP
name: http
port: 6379
targetPort: 6379
---
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:

36
kubernetes/servicemesh/applications/playlists-db/appendonly.aof Normal file
View File

@ -0,0 +1,36 @@
*2
$6
SELECT
$1
0
*3
$3
set
$9
playlists
$515
[
{
"id" : "1",
"name" : "CI/CD",
"videos": [ { "id" : "OFgziggbCOg"}, { "id" : "myCcJJ_Fk10"}, { "id" : "2WSJF7d8dUg"}]
},
{
"id" : "2",
"name" : "K8s in the Cloud",
"videos": [ { "id" : "QThadS3Soig"}, { "id" : "eyvLwK5C2dw"}]
},
{
"id" : "3",
"name" : "Storage and MessageBrokers",
"videos": [ { "id" : "JmCn7k0PlV4"}, { "id" : "_lpDfMkxccc"}]
},
{
"id" : "4",
"name" : "K8s Autoscaling",
"videos": [ { "id" : "jM36M39MA3I"}, { "id" : "FfDI08sgrYY"}]
}
]

BIN
kubernetes/servicemesh/applications/playlists-db/configmap.yaml Normal file
View File

Binary file not shown.

60
kubernetes/servicemesh/applications/playlists-db/deploy.yaml Normal file
View File

@ -0,0 +1,60 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: playlists-db
labels:
app: playlists-db
spec:
selector:
matchLabels:
app: playlists-db
replicas: 1
template:
metadata:
labels:
app: playlists-db
spec:
initContainers:
- name: data
image: busybox
command: [ "sh", "-c" ]
args:
- |
cp /config/appendonly.aof /tmp/appendonly.aof
volumeMounts:
- name: data
mountPath: /tmp/
- name: config
mountPath: /config/
containers:
- name: playlists-db
image: redis:6.0-alpine
ports:
- containerPort: 6379
volumeMounts:
- name: data
mountPath: /tmp/
command: [ "redis-server"]
args: ["--dir", "/tmp", "--appendonly", "yes"]
volumes:
- name: data
emptyDir: {}
- name: config
configMap:
name: playlists-db
---
apiVersion: v1
kind: Service
metadata:
name: playlists-db
labels:
app: playlists-db
spec:
type: ClusterIP
selector:
app: playlists-db
ports:
- protocol: TCP
name: http
port: 6379
targetPort: 6379

137
kubernetes/servicemesh/applications/videos-api/app.go
View File

@ -6,11 +6,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/go-redis/redis/v8"
"fmt"
"time"
"context"
"encoding/json"
"os"
"strconv"
"math/rand"
)
@ -46,8 +43,6 @@ func main() {
})
rdb = r
initDummyData()
fmt.Println("Running...")
log.Fatal(http.ListenAndServe(":10010", router))
}
@ -71,22 +66,6 @@ type stop struct {
error
}
func retry(attempts int, sleep time.Duration, fn func() error) error {
if err := fn(); err != nil {
if s, ok := err.(stop); ok {
// Return the original error for later checking
return s.error
}
if attempts--; attempts > 0 {
time.Sleep(sleep)
return retry(attempts, 2*sleep, fn)
}
return err
}
return nil
}
func cors(writer http.ResponseWriter) () {
if(environment == "DEBUG"){
writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
@ -96,122 +75,6 @@ func cors(writer http.ResponseWriter) () {
}
}
func initDummyData(){
//init dummy data into redis
duration, _ := time.ParseDuration("4s")
v1 := `{
"id" : "jM36M39MA3I",
"title" : "Kubernetes cluster autoscaling",
"imageurl" : "https://i.ytimg.com/vi/jM36M39MA3I/sddefault.jpg",
"url" : "https://youtu.be/jM36M39MA3I",
"description" : ""
}
`
v2 := `{
"id" : "FfDI08sgrYY",
"title" : "Kubernetes pod autoscaling",
"imageurl" : "https://i.ytimg.com/vi/FfDI08sgrYY/sddefault.jpg",
"url" : "https://youtu.be/FfDI08sgrYY",
"description" : ""
}
`
v3 := `{
"id" : "JmCn7k0PlV4",
"title" : "Redis on Kubernetes",
"imageurl" : "https://i.ytimg.com/vi/JmCn7k0PlV4/sddefault.jpg",
"url" : "https://youtu.be/JmCn7k0PlV4",
"description" : ""
}
`
v4 := `{
"id" : "_lpDfMkxccc",
"title" : "RabbitMQ on Kubernetes",
"imageurl" : "https://i.ytimg.com/vi/_lpDfMkxccc/sddefault.jpg",
"url" : "https://youtu.be/_lpDfMkxccc",
"description" : ""
}
`
v5 := `{
"id" : "OFgziggbCOg",
"title" : "Flux CD",
"imageurl" : "https://i.ytimg.com/vi/OFgziggbCOg/sddefault.jpg",
"url" : "https://youtu.be/OFgziggbCOg",
"description" : ""
}
`
v6 := `{
"id" : "myCcJJ_Fk10",
"title" : "Drone CI",
"imageurl" : "https://i.ytimg.com/vi/myCcJJ_Fk10/sddefault.jpg",
"url" : "https://youtu.be/myCcJJ_Fk10",
"description" : ""
}
`
v7 := `{
"id" : "2WSJF7d8dUg",
"title" : "Argo CD",
"imageurl" : "https://i.ytimg.com/vi/2WSJF7d8dUg/sddefault.jpg",
"url" : "https://youtu.be/2WSJF7d8dUg",
"description" : ""
}
`
v8 := `{
"id" : "QThadS3Soig",
"title" : "Kubernetes on Amazon",
"imageurl" : "https://i.ytimg.com/vi/QThadS3Soig/sddefault.jpg",
"url" : "https://youtu.be/QThadS3Soig",
"description" : ""
}
`
v9 := `{
"id" : "eyvLwK5C2dw",
"title" : "Kubernetes on Azure",
"imageurl" : "https://i.ytimg.com/vi/eyvLwK5C2dw/mqdefault.jpg?sqp=CISC_PoF&rs=AOn4CLDo7kizrJozB0pxBhxL9JbyiW_EPw",
"url" : "https://youtu.be/eyvLwK5C2dw",
"description" : ""
}`
dummy := []string{ v1, v2, v3, v4,v5, v6, v7, v8,v9 }
retry(10, duration, func() (err error){
for i := range dummy {
dummyItem := dummy[i]
v := videos{}
iStr := strconv.Itoa(i)
fmt.Println("checking vid: " + iStr)
err = json.Unmarshal([]byte(dummyItem), &v)
if err != nil {
break
}
fmt.Println("adding vid: " + v.Id)
err = rdb.Set(ctx, v.Id,dummyItem , 0).Err()
if err != nil {
break
}
}
if err != nil {
fmt.Println("error occured connecting to Redis, retrying...")
return err
}
fmt.Println("Redis dummy data initialised")
return nil
})
}
type videos struct {
Id string `json:"id"`
Title string `json:"title"`

37
kubernetes/servicemesh/applications/videos-api/deploy.yaml
View File

@ -51,42 +51,5 @@ spec:
port: 10010
targetPort: 10010
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: videos-db
labels:
app: videos-db
spec:
selector:
matchLabels:
app: videos-db
replicas: 1
template:
metadata:
labels:
app: videos-db
spec:
containers:
- name: videos-db
image: redis:6.0-alpine
ports:
- containerPort: 6379
---
apiVersion: v1
kind: Service
metadata:
name: videos-db
labels:
app: videos-db
spec:
type: ClusterIP
selector:
app: videos-db
ports:
- protocol: TCP
name: http
port: 6379
targetPort: 6379

130
kubernetes/servicemesh/applications/videos-db/appendonly.aof Normal file
View File

@ -0,0 +1,130 @@
*2
$6
SELECT
$1
0
*3
$3
set
$11
jM36M39MA3I
$206
{
"id" : "jM36M39MA3I",
"title" : "Kubernetes cluster autoscaling",
"imageurl" : "https://i.ytimg.com/vi/jM36M39MA3I/sddefault.jpg",
"url" : "https://youtu.be/jM36M39MA3I",
"description" : ""
}
*3
$3
set
$11
FfDI08sgrYY
$202
{
"id" : "FfDI08sgrYY",
"title" : "Kubernetes pod autoscaling",
"imageurl" : "https://i.ytimg.com/vi/FfDI08sgrYY/sddefault.jpg",
"url" : "https://youtu.be/FfDI08sgrYY",
"description" : ""
}
*3
$3
set
$11
JmCn7k0PlV4
$195
{
"id" : "JmCn7k0PlV4",
"title" : "Redis on Kubernetes",
"imageurl" : "https://i.ytimg.com/vi/JmCn7k0PlV4/sddefault.jpg",
"url" : "https://youtu.be/JmCn7k0PlV4",
"description" : ""
}
*3
$3
set
$11
_lpDfMkxccc
$198
{
"id" : "_lpDfMkxccc",
"title" : "RabbitMQ on Kubernetes",
"imageurl" : "https://i.ytimg.com/vi/_lpDfMkxccc/sddefault.jpg",
"url" : "https://youtu.be/_lpDfMkxccc",
"description" : ""
}
*3
$3
set
$11
OFgziggbCOg
$183
{
"id" : "OFgziggbCOg",
"title" : "Flux CD",
"imageurl" : "https://i.ytimg.com/vi/OFgziggbCOg/sddefault.jpg",
"url" : "https://youtu.be/OFgziggbCOg",
"description" : ""
}
*3
$3
set
$11
myCcJJ_Fk10
$184
{
"id" : "myCcJJ_Fk10",
"title" : "Drone CI",
"imageurl" : "https://i.ytimg.com/vi/myCcJJ_Fk10/sddefault.jpg",
"url" : "https://youtu.be/myCcJJ_Fk10",
"description" : ""
}
*3
$3
set
$11
2WSJF7d8dUg
$183
{
"id" : "2WSJF7d8dUg",
"title" : "Argo CD",
"imageurl" : "https://i.ytimg.com/vi/2WSJF7d8dUg/sddefault.jpg",
"url" : "https://youtu.be/2WSJF7d8dUg",
"description" : ""
}
*3
$3
set
$11
QThadS3Soig
$196
{
"id" : "QThadS3Soig",
"title" : "Kubernetes on Amazon",
"imageurl" : "https://i.ytimg.com/vi/QThadS3Soig/sddefault.jpg",
"url" : "https://youtu.be/QThadS3Soig",
"description" : ""
}
*3
$3
set
$11
eyvLwK5C2dw
$244
{
"id" : "eyvLwK5C2dw",
"title" : "Kubernetes on Azure",
"imageurl" : "https://i.ytimg.com/vi/eyvLwK5C2dw/mqdefault.jpg?sqp=CISC_PoF&rs=AOn4CLDo7kizrJozB0pxBhxL9JbyiW_EPw",
"url" : "https://youtu.be/eyvLwK5C2dw",
"description" : ""
}

BIN
kubernetes/servicemesh/applications/videos-db/configmap.yaml Normal file
View File

Binary file not shown.

60
kubernetes/servicemesh/applications/videos-db/deploy.yaml Normal file
View File

@ -0,0 +1,60 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: videos-db
labels:
app: videos-db
spec:
selector:
matchLabels:
app: videos-db
replicas: 1
template:
metadata:
labels:
app: videos-db
spec:
initContainers:
- name: data
image: busybox
command: [ "sh", "-c" ]
args:
- |
cp /config/appendonly.aof /tmp/appendonly.aof
volumeMounts:
- name: data
mountPath: /tmp/
- name: config
mountPath: /config/
containers:
- name: videos-db
image: redis:6.0-alpine
ports:
- containerPort: 6379
volumeMounts:
- name: data
mountPath: /tmp/
command: [ "redis-server"]
args: ["--dir", "/tmp", "--appendonly", "yes"]
volumes:
- name: data
emptyDir: {}
- name: config
configMap:
name: videos-db
---
apiVersion: v1
kind: Service
metadata:
name: videos-db
labels:
app: videos-db
spec:
type: ClusterIP
selector:
app: videos-db
ports:
- protocol: TCP
name: http
port: 6379
targetPort: 6379

6
kubernetes/servicemesh/docker-compose.yaml
View File

@ -32,6 +32,12 @@ services:
videos-db:
container_name: videos-db
image: redis:6.0-alpine
command: [ "redis-server" , "--dir", "/tmp", "--appendonly", "yes"]
volumes:
- ./applications/videos-db/appendonly.aof:/tmp/appendonly.aof
playlists-db:
container_name: playlists-db
image: redis:6.0-alpine
command: [ "redis-server" , "--dir", "/tmp", "--appendonly", "yes"]
volumes:
- ./applications/playlists-db/appendonly.aof:/tmp/appendonly.aof

38
kubernetes/servicemesh/introduction.md
View File

@ -111,24 +111,33 @@ This is intentional to demonstrate a busy network.
## Adding an Ingress Controller
Adding an ingress controller allows us to route all our traffic. </br>
We setup a `host` file with entry `demo.com 127.0.0.1`
We setup a `host` file with entry `127.0.0.1 servicemesh.demo`
And `port-forward` to the `ingress-controller`
```
demo.com/home --> videos-web
demo.com/api/playlists --> playlists-api
servicemesh.demo/home --> videos-web
servicemesh.demo/api/playlists --> playlists-api
servicemesh.demo/home +--------------+
+------------------------------> | videos-web |
| | |
servicemesh.demo/home +------+------------+ +--------------+
+------------------>+ingress-nginx |
|Ingress controller |
+------+------------+ +---------------+ +--------------+
| | playlists-api +--->+ playlists-db |
+------------------------------> | | | |
servicemesh.demo/api/playlists +-----+---------+ +--------------+
|
v
+-----+------+ +-----------+
| videos-api +------>+ videos-db |
| | | |
+------------+ +-----------+
+-------------------+ demo.com/home +------------+ +---------------+ +--------------+
|ingress-nginx +----------------->+ videos+web +---->+ playlists+api +--->+ playlists+db |
|Ingress controller | | | | | | |
+---------+---------+ +------------+ +-----+---------+ +--------------+
| |
| v
| demo.com api/playlists +-----+------+ +-----------+
+---------------------------------------------> | videos+api +------>+ videos+db |
| | | |
+------------+ +-----------+
```
<br/>
@ -188,6 +197,7 @@ It's blank because it needs the `playlists-api` to get data
cd ./kubernetes/servicemesh/
kubectl apply -f applications/playlists-api/deploy.yaml
kubectl apply -f applications/playlists-db/
kubectl port-forward svc/playlists-api 81:80
```
@ -206,7 +216,7 @@ Playlists are empty because it needs the `videos-api` to get video data <br/>
cd ./kubernetes/servicemesh/
kubectl apply -f applications/videos-api/deploy.yaml
kubectl apply -f applications/videos-db/
```
Refresh page at `http://localhost/` <br/>

41
kubernetes/servicemesh/linkerd/readme.md
View File

@ -5,7 +5,7 @@
Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
```
kind create cluster --name linkerd --image kindest/node:v1.18.4
kind create cluster --name linkerd --image kindest/node:v1.19.1
```
## Deploy our microservices (Video catalog)
@ -17,8 +17,10 @@ kubectl apply -f kubernetes/servicemesh/applications/ingress-nginx/
# applications
kubectl apply -f kubernetes/servicemesh/applications/playlists-api/
kubectl apply -f kubernetes/servicemesh/applications/playlists-db/
kubectl apply -f kubernetes/servicemesh/applications/videos-api/
kubectl apply -f kubernetes/servicemesh/applications/videos-web/
kubectl apply -f kubernetes/servicemesh/applications/videos-db/
```
## Make sure our applications are running
@ -94,12 +96,12 @@ linkerd-control-plane Ready master 26m v1.18.4
## Linkerd CLI
Lets download the `linkerd` command line tool <br/>
I grabbed the `edge-20.9.4` release using `curl`
I grabbed the `edge-20.10.1` release using `curl`
You can go to the [releases](https://github.com/linkerd/linkerd2/releases/tag/edge-20.9.4) page to get it
You can go to the [releases](https://github.com/linkerd/linkerd2/releases/tag/edge-20.10.1) page to get it
```
curl -L -o linkerd https://github.com/linkerd/linkerd2/releases/download/edge-20.9.4/linkerd2-cli-edge-20.9.4-linux-amd64
curl -L -o linkerd https://github.com/linkerd/linkerd2/releases/download/edge-20.10.1/linkerd2-cli-edge-20.10.1-linux-amd64
chmod +x linkerd && mv ./linkerd /usr/local/bin/
linkerd --help
@ -117,19 +119,19 @@ linkerd check --pre
## Get the YAML
```
linkerd install > ./kubernetes/servicemesh/linkerd/manifest/linkerd-edge-20.9.4.yaml
linkerd install > ./kubernetes/servicemesh/linkerd/manifest/linkerd-edge-20.10.1.yaml
```
## Install Linkerd
```
kubectl apply -f ./kubernetes/servicemesh/linkerd/manifest/linkerd-edge-20.9.4.yaml
kubectl apply -f ./kubernetes/servicemesh/linkerd/manifest/linkerd-edge-20.10.1.yaml
```
Let's wait until all components are running
```
kubectl -n linkerd get deploy
watch kubectl -n linkerd get pods
kubectl -n linkerd get svc
```
@ -178,6 +180,7 @@ kubectl get deploy playlists-db -o yaml | linkerd inject - | kubectl apply -f -
kubectl get deploy videos-api -o yaml | linkerd inject - | kubectl apply -f -
kubectl get deploy videos-db -o yaml | linkerd inject - | kubectl apply -f -
kubectl get deploy videos-web -o yaml | linkerd inject - | kubectl apply -f -
kubectl -n ingress-nginx get deploy nginx-ingress-controller -o yaml | linkerd inject - | kubectl apply -f -
```
@ -204,6 +207,7 @@ kubectl edit deploy videos-api
#set environment FLAKY=true
```
# Service Profile
linkerd profile -n default videos-api --tap deploy/videos-api --tap-duration 10s
linkerd profile -n default videos-api --template
@ -213,4 +217,27 @@ We can see that service profile helps us add retry policies in place: <br/>
```
linkerd routes -n default deploy/playlists-api --to svc/videos-api -o wide
linkerd top deploy/videos-api
```
# Mutual TLS
We can validate if mTLS is working
```
/work # linkerd -n default edges deployment
SRC DST SRC_NS DST_NS SECURED
playlists-api videos-api default default √
linkerd-prometheus playlists-api linkerd default √
linkerd-prometheus playlists-db linkerd default √
linkerd-prometheus videos-api linkerd default √
linkerd-prometheus videos-db linkerd default √
linkerd-prometheus videos-web linkerd default √
linkerd-tap playlists-api linkerd default √
linkerd-tap playlists-db linkerd default √
linkerd-tap videos-api linkerd default √
linkerd-tap videos-db linkerd default √
linkerd-tap videos-web linkerd default √
linkerd -n default tap deploy
```

5
kubernetes/servicemesh/linkerd/serviceprofiles/videos-api.yaml
View File

@ -1,7 +1,6 @@
apiVersion: linkerd.io/v1alpha2
kind: ServiceProfile
metadata:
creationTimestamp: null
name: videos-api.default.svc.cluster.local
namespace: default
spec:
@ -9,5 +8,5 @@ spec:
- condition:
method: GET
pathRegex: /.*
isRetryable: true
name: GET *
name: AUTO RETRY ALL
isRetryable: true