From 7efaec5886f1fe09a8ac216b9961b75af53bcc8d Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 6 Apr 2018 15:45:22 +0100 Subject: [PATCH 1/2] Workaround `rootfsPropagation: shared` brokenness. Despite using `rootfsPropagation` after updating to Kubernetes v1.10 (not done here yet) we are seeing: RunContainerError: failed to start container 4ea4bde9b43a9eb241a5d7d98abf87184938f85ce9139949a3a246b6fe6b8985": Error response from daemon: linux mounts: path /etc/kubernetes/pki/etcd is mounted on /etc/kubernetes but it is not a shared or slave mount Workaround this by temporarily (re)doing it in the entrypoint. Signed-off-by: Ian Campbell --- pkg/cri-containerd/Dockerfile | 4 +++- yml/docker.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/cri-containerd/Dockerfile b/pkg/cri-containerd/Dockerfile index 0d7d42e..12f7aa5 100644 --- a/pkg/cri-containerd/Dockerfile +++ b/pkg/cri-containerd/Dockerfile @@ -47,5 +47,7 @@ RUN make DESTDIR=/out install FROM scratch WORKDIR / -ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/opt/cni/bin", "--network-conf-dir", "/etc/cni/net.d"] +# `rootfsPropagation: shared` (used in `build.yml`) appears to be broken at the moment, workaround that issue here. +#ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/opt/cni/bin", "--network-conf-dir", "/etc/cni/net.d"] +ENTRYPOINT ["/bin/sh", "-c", "set -ex; mount --make-shared / && exec cri-containerd -v 2 --alsologtostderr --network-bin-dir /opt/cni/bin --network-conf-dir /etc/cni/net.d"] COPY --from=build /out / diff --git a/yml/docker.yml b/yml/docker.yml index fdd1ea1..b53f5b8 100644 --- a/yml/docker.yml +++ b/yml/docker.yml @@ -21,7 +21,9 @@ services: - /var/lib/cni/bin:/opt/cni/bin:rshared,rbind - /var/lib/kubelet-plugins:/usr/libexec/kubernetes/kubelet-plugins:rshared,rbind rootfsPropagation: shared - command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] + # `rootfsPropagation: shared` appears to be broken, workaround that issue here. + #command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] + command: ["/bin/sh", "-c", "set -ex; mount --make-shared / && exec /usr/local/bin/docker-init /usr/local/bin/dockerd"] runtime: mkdir: ["/var/lib/kubeadm", "/var/lib/cni/conf", "/var/lib/cni/bin", "/var/lib/kubelet-plugins"] cgroupsPath: podruntime/docker From a060e6278d1bd32728ebb089267f7011ab4d44f5 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 6 Apr 2018 16:01:13 +0100 Subject: [PATCH 2/2] Update to Kubernetes v1.10.0. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This also required updating some image versions in the caches: kube_dns_version: 1.14.7 → 1.14.8 pause_version: 3.0 → 3.1 etcd_version: 3.1.11 → 3.1.12 From https://kubernetes.io/docs/imported/release/notes/#before-upgrading: [action-required] The Container Runtime Interface (CRI) version has increased from v1alpha1 to v1alpha2. Runtimes implementing the CRI will need to update to the new version, which configures container namespaces using an enumeration rather than booleans. (#58973, @verb) Thus we must update to a newer cri-containerd in lock-step. However this is made more complicated because cri-containerd is no longer a standalone daemon but has become a containerd plugin, which will require some rearchitecting (and which is so far only part of containerd v1.1-rc). Luckily the version right before standalone mode was deleted is available in a branch and supports the required API version so switch to that here as a stop gap measure. Note that the naming has changed kubernetes-incubator/cri-containerd → containerd/cri-containerd → containerd/cri with the branch we are using being at the middle step so update the naming and paths appropriately. The command line options have changed a bit, so adjust. Signed-off-by: Ian Campbell --- pkg/cri-containerd/Dockerfile | 17 +++++++++-------- pkg/kube-e2e-test/Dockerfile | 2 +- pkg/kubelet/Dockerfile | 2 +- .../images.lst | 10 +++++----- .../images.lst | 8 ++++---- scripts/mk-image-cache-lst | 8 ++++---- yml/cri-containerd.yml | 2 +- yml/docker-master.yml | 2 +- yml/docker.yml | 2 +- yml/kube.yml | 2 +- 10 files changed, 28 insertions(+), 27 deletions(-) diff --git a/pkg/cri-containerd/Dockerfile b/pkg/cri-containerd/Dockerfile index 12f7aa5..84144be 100644 --- a/pkg/cri-containerd/Dockerfile +++ b/pkg/cri-containerd/Dockerfile @@ -13,13 +13,14 @@ RUN \ && true ENV GOPATH=/go PATH=$PATH:/go/bin -ENV CRI_CONTAINERD_URL https://github.com/kubernetes-incubator/cri-containerd.git +ENV CRI_CONTAINERD_URL https://github.com/containerd/cri-containerd.git #ENV CRI_CONTAINERD_BRANCH pull/NNN/head -ENV CRI_CONTAINERD_COMMIT v1.0.0-beta.1 -RUN mkdir -p $GOPATH/src/github.com/kubernetes-incubator && \ - cd $GOPATH/src/github.com/kubernetes-incubator && \ +# This is the `standalone-cri-containerd` branch, which is at the point right before standalone mode was deleted. +ENV CRI_CONTAINERD_COMMIT 64b098a293831f742aeb3dd3e48a5405990c14c5 +RUN mkdir -p $GOPATH/src/github.com/containerd && \ + cd $GOPATH/src/github.com/containerd && \ git clone $CRI_CONTAINERD_URL cri-containerd -WORKDIR $GOPATH/src/github.com/kubernetes-incubator/cri-containerd +WORKDIR $GOPATH/src/github.com/containerd/cri-containerd RUN set -e; \ if [ -n "$CRI_CONTAINERD_BRANCH" ] ; then \ git fetch origin "$CRI_CONTAINERD_BRANCH"; \ @@ -31,7 +32,7 @@ RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ # util-linux because a full ns-enter is required. # example commands: /usr/bin/nsenter --net= -F -- # /usr/bin/nsenter --net=/var/run/netns/cni-5e8acebe-810d-c1b9-ced0-47be2f312fa8 -F -- -# NB the first ("--net=") is actually not valid -- see https://github.com/kubernetes-incubator/cri-containerd/issues/245 +# NB the first ("--net=") is actually not valid -- see https://github.com/containerd/cri/issues/245 RUN apk add --no-cache --initdb -p /out \ alpine-baselayout \ busybox \ @@ -48,6 +49,6 @@ RUN make DESTDIR=/out install FROM scratch WORKDIR / # `rootfsPropagation: shared` (used in `build.yml`) appears to be broken at the moment, workaround that issue here. -#ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/opt/cni/bin", "--network-conf-dir", "/etc/cni/net.d"] -ENTRYPOINT ["/bin/sh", "-c", "set -ex; mount --make-shared / && exec cri-containerd -v 2 --alsologtostderr --network-bin-dir /opt/cni/bin --network-conf-dir /etc/cni/net.d"] +#ENTRYPOINT ["cri-containerd", "--log-level", "info", "--network-bin-dir", "/opt/cni/bin", "--network-conf-dir", "/etc/cni/net.d"] +ENTRYPOINT ["/bin/sh", "-c", "set -ex; mount --make-shared / && exec cri-containerd --log-level info --network-bin-dir /opt/cni/bin --network-conf-dir /etc/cni/net.d"] COPY --from=build /out / diff --git a/pkg/kube-e2e-test/Dockerfile b/pkg/kube-e2e-test/Dockerfile index 93ede7e..1a2d3ec 100644 --- a/pkg/kube-e2e-test/Dockerfile +++ b/pkg/kube-e2e-test/Dockerfile @@ -3,7 +3,7 @@ FROM linuxkit/alpine:f3cd219615428b2bd943411723eb28875275fae7 AS build # When changing kubernetes_version remember to also update: # - scripts/mk-image-cache-lst and run `make refresh-image-caches` from top-level # - pkg/kubelet/Dockerfile -ENV kubernetes_version v1.9.6 +ENV kubernetes_version v1.10.0 RUN apk add -U --no-cache \ bash \ diff --git a/pkg/kubelet/Dockerfile b/pkg/kubelet/Dockerfile index f2b0ada..7869049 100644 --- a/pkg/kubelet/Dockerfile +++ b/pkg/kubelet/Dockerfile @@ -3,7 +3,7 @@ FROM linuxkit/alpine:f3cd219615428b2bd943411723eb28875275fae7 AS build # When changing kubernetes_version remember to also update: # - scripts/mk-image-cache-lst and run `make refresh-image-caches` from top-level # - pkg/e2e-test/Dockerfile -ENV kubernetes_version v1.9.6 +ENV kubernetes_version v1.10.0 ENV cni_version v0.7.0 ENV critools_version v1.0.0-alpha.0 diff --git a/pkg/kubernetes-docker-image-cache-common/images.lst b/pkg/kubernetes-docker-image-cache-common/images.lst index 2bc5fb5..0af540a 100644 --- a/pkg/kubernetes-docker-image-cache-common/images.lst +++ b/pkg/kubernetes-docker-image-cache-common/images.lst @@ -1,7 +1,7 @@ # autogenerated by: # ./scripts/mk-image-cache-lst common -gcr.io/google_containers/kube-proxy-amd64:v1.9.6@sha256:dd647b619b75882f217a21a6e2c473deb77cba1cfb20461da0084ed78d951af1 -gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.7@sha256:f80f5f9328107dc516d67f7b70054354b9367d31d4946a3bffd3383d83d7efe8 -gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.7@sha256:f5bddc71efe905f4e4b96f3ca346414be6d733610c1525b98fff808f93966680 -gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.7@sha256:6cfb9f9c2756979013dbd3074e852c2d8ac99652570c5d17d152e0c0eb3321d6 -gcr.io/google_containers/pause-amd64:3.0@sha256:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516 +gcr.io/google_containers/kube-proxy-amd64:v1.10.0@sha256:fc944b06c14cb442916045a630d5e374dfb9c453dfc56d3cb59ac21ea4268875 +gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.8@sha256:23df717980b4aa08d2da6c4cfa327f1b730d92ec9cf740959d2d5911830d82fb +gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.8@sha256:6d8e0da4fb46e9ea2034a3f4cab0e095618a2ead78720c12e791342738e5f85d +gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.8@sha256:93c827f018cf3322f1ff2aa80324a0306048b0a69bc274e423071fb0d2d29d8b +gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610 diff --git a/pkg/kubernetes-docker-image-cache-control-plane/images.lst b/pkg/kubernetes-docker-image-cache-control-plane/images.lst index 788ce64..e5e280b 100644 --- a/pkg/kubernetes-docker-image-cache-control-plane/images.lst +++ b/pkg/kubernetes-docker-image-cache-control-plane/images.lst @@ -1,6 +1,6 @@ # autogenerated by: # ./scripts/mk-image-cache-lst control-plane -gcr.io/google_containers/kube-apiserver-amd64:v1.9.6@sha256:1bb16ddef9edd8142125ac2443e36986c051b0df808db907648ac9d68e0d5f23 -gcr.io/google_containers/kube-controller-manager-amd64:v1.9.6@sha256:c90d1f38999b96405a997e52392b4db2ac68b6d4ea376e2e99ab528c51b88d22 -gcr.io/google_containers/kube-scheduler-amd64:v1.9.6@sha256:0bc20f6ee28be19c4c45f2a5d0282e7ae3e89d61789e7a89182f818b37b67e28 -gcr.io/google_containers/etcd-amd64:3.1.11@sha256:54889c08665d241e321ca5ce976b2df0f766794b698d53faf6b7dacb95316680 +gcr.io/google_containers/kube-apiserver-amd64:v1.10.0@sha256:8b7a675c6fdda0469e971e5b1f3e902bb71c36396faf506f1511052705a5b0ee +gcr.io/google_containers/kube-controller-manager-amd64:v1.10.0@sha256:ab637b1f4700e0cc83d3ae3c5368919eb8f6199485adcc126fd306481df56fec +gcr.io/google_containers/kube-scheduler-amd64:v1.10.0@sha256:4670717a1c092680dc0ac018dc36f991e06ffbee6b6bafba9a2968d7bebe7800 +gcr.io/google_containers/etcd-amd64:3.1.12@sha256:68235934469f3bc58917bcf7018bf0d3b72129e6303b0bef28186d96b2259317 diff --git a/scripts/mk-image-cache-lst b/scripts/mk-image-cache-lst index b39b119..6949023 100755 --- a/scripts/mk-image-cache-lst +++ b/scripts/mk-image-cache-lst @@ -3,10 +3,10 @@ repo=gcr.io/google_containers # When changing kubernetes_version remember to also update: # - pkg/kubelet/Dockerfile # - pkg/e2e-test/Dockerfile -kubernetes_version=v1.9.6 -kube_dns_version=1.14.7 -pause_version=3.0 -etcd_version=3.1.11 +kubernetes_version=v1.10.0 +kube_dns_version=1.14.8 +pause_version=3.1 +etcd_version=3.1.12 common=" kube-proxy-amd64:$kubernetes_version diff --git a/yml/cri-containerd.yml b/yml/cri-containerd.yml index 4f93e2d..91f054e 100644 --- a/yml/cri-containerd.yml +++ b/yml/cri-containerd.yml @@ -1,6 +1,6 @@ services: - name: cri-containerd - image: linuxkit/cri-containerd:c7dfc51dfb34aa4df01beb23069e1cadb0bb1bb7 + image: linuxkit/cri-containerd:474d704f6dcb173b10768ab1d1bd61674468c7f9 cgroupsPath: podruntime/cri-containerd files: - path: /etc/kubelet.sh.conf diff --git a/yml/docker-master.yml b/yml/docker-master.yml index 463a859..f4c5570 100644 --- a/yml/docker-master.yml +++ b/yml/docker-master.yml @@ -1,4 +1,4 @@ services: - name: kubernetes-docker-image-cache-control-plane - image: linuxkit/kubernetes-docker-image-cache-control-plane:b1bddee9fd3cfb73fad12408d5ca08136fd77889 + image: linuxkit/kubernetes-docker-image-cache-control-plane:e41541ca040a0001e02dd5c8ac40e0ae38110a7a cgroupsPath: podruntime/control-cache diff --git a/yml/docker.yml b/yml/docker.yml index b53f5b8..858fb0e 100644 --- a/yml/docker.yml +++ b/yml/docker.yml @@ -28,7 +28,7 @@ services: mkdir: ["/var/lib/kubeadm", "/var/lib/cni/conf", "/var/lib/cni/bin", "/var/lib/kubelet-plugins"] cgroupsPath: podruntime/docker - name: kubernetes-docker-image-cache-common - image: linuxkit/kubernetes-docker-image-cache-common:11e096a191cf68ec047ec74985951d225af18197 + image: linuxkit/kubernetes-docker-image-cache-common:a4eb1313407e72fda5789b9ca604997650126539 cgroupsPath: podruntime/common-cache files: - path: /etc/kubelet.sh.conf diff --git a/yml/kube.yml b/yml/kube.yml index eac53e0..70a1f1d 100644 --- a/yml/kube.yml +++ b/yml/kube.yml @@ -40,7 +40,7 @@ services: image: linuxkit/sshd:39d6bdc9a7489ceffa761ad5cb96c87b50d6732d cgroupsPath: systemreserved/sshd - name: kubelet - image: linuxkit/kubelet:7e09d17481ac9dd43617d84d6b1b3af964ded505 + image: linuxkit/kubelet:7663c7ad197bf3d401d93d5bc4c0eabfa2e70f77 cgroupsPath: podruntime/kubelet files: - path: etc/linuxkit.yml