作者：張華 發表於：2020-03-13
版權聲明：可以任意轉載，轉載時請務必以超鏈接形式標明文章原始出處和作者信息及本版權聲明
set up a test private registry
docker pull registry:2
mkdir ~/registry/certs && cd ~/registry/certs
openssl genrsa -passout pass:password -out ca.key
openssl req -x509 -passin pass:password -new -nodes -key ca.key -days 3650 -out ca.crt -subj "/C=CN/ST=BJ/O=STS/CN=zhhuabj-bastion.cloud.sts"
#openssl genrsa -passout pass:password -out server.key
#openssl req -new -key server.key -out server.csr -subj "/C=CN/ST=BJ/O=STS/CN=zhhuabj-bastion.cloud.sts"
#openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 3650
#cat server.crt server.key > server.pem
cd ~/registry && mkdir auth && mkdir images
sudo docker run --entrypoint htpasswd registry:2 -Bbn test password > ./auth/htpasswd
sudo docker run -d -p 5000:5000 -v `pwd`/images:/var/lib/registry --restart=always --name registry \
-v `pwd`/auth:/auth \
-e "REGISTRY_AUTH=htpasswd" \
-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-v `pwd`/certs:/certs \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/ca.crt \
-e REGISTRY_HTTP_TLS_KEY=/certs/ca.key \
registry:2
curl -X GET --cacert /etc/docker/certs.d/zhhuabj-bastion.cloud.sts:5000/ca.crt https://zhhuabj-bastion.cloud.sts:5000
upload a test image into test private registry
scp -r [email protected]:/root/registry/certs/ca.crt .
sudo mkdir -p /etc/docker/certs.d/zhhuabj-bastion.cloud.sts:5000
sudo cp ca.crt /etc/docker/certs.d/zhhuabj-bastion.cloud.sts:5000/
#must restart docker, or it will throw x509 related errors when running 'docker push'
sudo systemctl restart docker
#another error when running 'docker login' - cannot autolaunch D-Bus without X11 $DISPLAY
#sudo apt autoremove --purge docker-compose -y
sudo docker login -u=test -p=password zhhuabj-bastion.cloud.sts:5000
# cat ~/.docker/config.json
{
"auths": {
"https://index.docker.io/v1/": {
"auth": "emhodWFiajpuYW50aW5nNDEz"
},
"zhhuabj-bastion.cloud.sts:5000": {
"auth": "dGVzdDpwYXNzd29yZA=="
}
},
"HttpHeaders": {
"User-Agent": "Docker-Client/18.09.7 (linux)"
}
}
sudo docker pull busybox && sudo docker tag docker.io/busybox zhhuabj-bastion.cloud.sts:5000/busybox
sudo docker push zhhuabj-bastion.cloud.sts:5000/busybox
sudo docker logout zhhuabj-bastion.cloud.sts:5000
#docker save zhhuabj-bastion.cloud.sts:5000/busybox > busybox.tar
curl -X GET --cacert /etc/containerd/ca.crt https://zhhuabj-bastion.cloud.sts:5000/v2/busybox/manifests/latest --user test:password
test it in docker
sudo cp ca.crt /etc/docker/certs.d/zhhuabj-bastion.cloud.sts:5000/
sudo systemctl restart docker
sudo docker login -u=test -p=password zhhuabj-bastion.cloud.sts:5000
sudo docker pull zhhuabj-bastion.cloud.sts:5000/busybox
test it in containerd
sudo apt install containerd #it will stop docker automatically
sudo mkdir -p /etc/containerd
containerd config default |sudo tee /etc/containerd/config.toml
#juju scp ./certs/ca.crt kubernetes-worker/1:/home/ubuntu/
sudo vim /etc/containerd/config.toml
change
[plugins.cri.registry]
[plugins.cri.registry.mirrors]
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://registry-1.docker.io"]
to
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://registry-1.docker.io"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."zhhuabj-bastion.cloud.sts:5000"]
endpoint = ["zhhuabj-bastion.cloud.sts:5000"]
[plugins."io.containerd.grpc.v1.cri".registry.auths]
[plugins."io.containerd.grpc.v1.cri".registry.auths."https://zhhuabj-bastion.cloud.sts:5000"]
auth = "dGVzdDpwYXNzd29yZA=="
username = "test"
password = "password"
[plugins."io.containerd.grpc.v1.cri".registry.tls_configs]
[plugins."io.containerd.grpc.v1.cri".registry.tls_configs."zhhuabj-bastion.cloud.sts:5000"]
ca_file = "/etc/containerd/ca.crt"
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.configs."zhhuabj-bastion.cloud.sts:5000"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."zhhuabj-bastion.cloud.sts:5000".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."zhhuabj-bastion.cloud.sts:5000".auth]
auth = "dGVzdDpwYXNzd29yZA=="
username = "test"
password = "password"
sudo systemctl restart containerd
#sudo ctr -n k8s.io image import busybox.tar
curl -X GET --cacert /etc/containerd/ca.crt https://zhhuabj-bastion.cloud.sts:5000/v2/busybox/manifests/latest --user test:password
echo -n | openssl s_client -showcerts -connect zhhuabj-bastion.cloud.sts:5000
sudo cp /home/ubuntu/ca.crt /etc/containerd/
sudo ctr --debug images pull --user test:password zhhuabj-bastion.cloud.sts:5000/busybox:latest --skip-verify
sudo ctr --namespace k8s.io image ls |grep busybox
sudo ctr image ls |grep zhhuabj
#seems there is a bug with insecure_skip_verify=true, finally we run update-ca-certificates, then remove 'skip-verify'
cp /etc/containerd/ca.crt /usr/local/share/ca-certificates/
update-ca-certificates
#sudo ctr --debug images pull --user test:password zhhuabj-bastion.cloud.sts:5000/busybox:latest --skip-verify
sudo ctr --debug images pull --user test:password zhhuabj-bastion.cloud.sts:5000/busybox:latest
test it in k8s
Before running the following commands, we need first to modify above /etc/containerd/config.toml, custom_registries of containerd charm will do it
juju config containerd custom_registries='[{"url": "https://zhhuabj-bastion.cloud.sts:5000", "username": "test", "password": "password"}]'
But how to run update-ca-certificates with ca.crt ? we search ‘ca_crt_path’ option from charm code. but how to do ?
then we can run:
#kubectl create secret docker-registry regcred --docker-server=zhhuabj-bastion.cloud.sts:5000 --docker-username=test --docker-password=password --docker-email='[email protected]'
sudo bash -c 'cat >config.json' <<EOF
{
"auths": {
"zhhuabj-bastion.cloud.sts:5000": {
"auth": "dGVzdDpwYXNzd29yZA=="
}
},
"HttpHeaders": {
"User-Agent": "Docker-Client/18.09.7 (linux)"
}
}
EOF
kubectl create secret generic regcred \
--from-file=.dockerconfigjson=./config.json \
--type=kubernetes.io/dockerconfigjson
kubectl get secret regcred --output="jsonpath={.data.\.dockerconfigjson}" | base64 --decode
sudo bash -c 'cat >busybox.yaml' <<EOF
apiVersion: v1
kind: Pod
metadata:
name: busybox
namespace: default
spec:
containers:
- name: busybox
image: zhhuabj-bastion.cloud.sts:5000/busybox:latest
command:
- sleep
- "3600"
imagePullSecrets:
- name: regcred
restartPolicy: Always
EOF
kubectl create -f busybox.yaml
kubectl get events
appendix - create another test image
sudo bash -c 'cat >simple-http-server.py' <<EOF
import SimpleHTTPServer
import SocketServer
PORT = 8000
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("0.0.0.0", PORT), Handler)
print "serving at port", PORT
httpd.serve_forever()
EOF
sudo bash -c 'cat >Dockerfile' <<EOF
FROM python:2.7-alpine
ADD simple-http-server.py /
RUN apk update && apk add bash
RUN pip install simple_http_server
CMD [ "python", "./simple-http-server.py" ]
EOF
sudo docker build -t simple-http-server .
sudo docker tag simple-http-server zhhuabj-bastion.cloud.sts:5000/simple-http-server
sudo docker login -u=test -p=password zhhuabj-bastion.cloud.sts:5000
sudo docker push zhhuabj-bastion.cloud.sts/simple-http-server
sudo docker logout zhhuabj-bastion.cloud.sts:5000
reference
[1] https://itnext.io/working-with-image-registries-and-containerd-in-kubernetes-63c311b86368
[2] https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
[3] https://www.digitalocean.com/community/tutorials/how-to-set-up-a-private-docker-registry-on-ubuntu-18-04
[4] https://kubernetes.io/zh/docs/setup/production-environment/container-runtimes/
[5] https://github.com/containerd/cri/blob/master/docs/registry.md#configure-registry-tls-communication
[6] https://blog.csdn.net/y_chen_007/article/details/97525206