maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Implement Docker healthcheck script in Go (#7105)

Browse source 
Go script makes it easy to read/maintain. Also updated the timeout
in Dockerfiles from 5s to default 30s and test interval to 1m

Higher timeout makes sense as server may sometimes respond slowly
if under high load as reported in #6974

Fixes #6974
This commit is contained in:
Nitish Tiwari 2019-02-20 21:42:03 +05:30 committed by GitHub
parent ce960565b1
commit 1e82c4a7c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 198 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -22,4 +22,5 @@ parts/
prime/
stage/
.sia_temp/
config.json
config.json
healthcheck

9
Dockerfile
View file

@ -11,7 +11,8 @@ RUN \
apk add --no-cache git && \
go get -v -d github.com/minio/minio && \
cd /go/src/github.com/minio/minio && \
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
go build -ldflags "-s -w" -o /usr/bin/healthcheck dockerscripts/healthcheck.go
FROM alpine:3.7
@ -22,7 +23,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
EXPOSE 9000
COPY --from=0 /go/bin/minio /usr/bin/minio
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' && \
@ -32,7 +34,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
HEALTHCHECK --interval=30s --timeout=5s \
CMD /usr/bin/healthcheck.sh
HEALTHCHECK --interval=1m CMD healthcheck
CMD ["minio"]

7
Dockerfile.dev
View file

@ -2,7 +2,7 @@ FROM alpine:3.7
LABEL maintainer="Minio Inc <dev@minio.io>"
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
COPY minio /usr/bin/
ENV MINIO_UPDATE off
@ -14,7 +14,7 @@ RUN \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh && \
chmod +x /usr/bin/healthcheck.sh
chmod +x /usr/bin/healthcheck
EXPOSE 9000
@ -22,7 +22,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
HEALTHCHECK --interval=30s --timeout=5s \
CMD /usr/bin/healthcheck.sh
HEALTHCHECK --interval=1m CMD healthcheck
CMD ["minio"]

21
Dockerfile.release
View file

@ -1,8 +1,22 @@
FROM golang:1.11.4-alpine3.7
ENV GOPATH /go
ENV CGO_ENABLED 0
WORKDIR /go/src/github.com/minio/
RUN \
apk add --no-cache git && \
go get -v -d github.com/minio/minio && \
cd /go/src/github.com/minio/minio/dockerscripts && \
go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
FROM alpine:3.7
LABEL maintainer="Minio Inc <dev@minio.io>"
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
@ -14,7 +28,7 @@ RUN \
curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh && \
chmod +x /usr/bin/healthcheck.sh
chmod +x /usr/bin/healthcheck
EXPOSE 9000
@ -22,7 +36,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
HEALTHCHECK --interval=30s --timeout=5s \
CMD /usr/bin/healthcheck.sh
HEALTHCHECK --interval=1m CMD healthcheck
CMD ["minio"]

1
Makefile
View file

@ -66,6 +66,7 @@ coverage: build
build: checks
@echo "Building minio binary to './minio'"
@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
@GOFLAGS="" CGO_ENABLED=0 go build --ldflags="-s -w" -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go
docker: build
@docker build -t $(TAG) . -f Dockerfile.dev

170
dockerscripts/healthcheck.go Executable file
View file

@ -0,0 +1,170 @@
/*
* Minio Cloud Storage, (C) 2019 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"bufio"
"crypto/tls"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/exec"
"strings"
"time"
xhttp "github.com/minio/minio/cmd/http"
)
const (
initGraceTime = 300
healthPath = "/minio/health/live"
timeout = time.Duration(30 * time.Second)
minioProcess = "minio"
)
// returns container boot time by finding
// modtime of /proc/1 directory
func getStartTime() time.Time {
di, err := os.Stat("/proc/1")
if err != nil {
// Cant stat proc dir successfully, exit with error
log.Fatal(err.Error())
}
return di.ModTime()
}
// Returns the ip:port of the Minio process
// running in the container
func findEndpoint() string {
cmd := exec.Command("netstat", "-ntlp")
stdout, err := cmd.StdoutPipe()
if err != nil {
// error getting stdout pipe
log.Fatal(err.Error())
}
if err = cmd.Start(); err != nil {
// error starting the command
log.Fatal(err.Error())
}
// split netstat output in rows
scanner := bufio.NewScanner(stdout)
scanner.Split(bufio.ScanLines)
// loop over the rows to find Minio process
for scanner.Scan() {
if strings.Contains(scanner.Text(), minioProcess) {
line := scanner.Text()
newLine := strings.Replace(line, ":::", "127.0.0.1:", 1)
fields := strings.Fields(newLine)
// index 3 in the row has the Local address
// find the last index of ":" - address will
// have port number after this index
i := strings.LastIndex(fields[3], ":")
// split address and port
addr := fields[3][:i]
port := fields[3][i+1:]
// add surrounding [] for ip6 address
if strings.Count(addr, ":") > 0 {
addr = strings.Join([]string{"[", addr, "]"}, "")
}
// return joint address and port
return strings.Join([]string{addr, port}, ":")
}
}
if err = cmd.Wait(); err != nil {
// command failed to run
log.Fatal(err.Error())
}
// minio process not found, exit with error
os.Exit(1)
return ""
}
func main() {
startTime := getStartTime()
// In distributed environment like Swarm, traffic is routed
// to a container only when it reports a `healthy` status. So, we exit
// with 0 to ensure healthy status till distributed Minio starts (120s).
// Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272
if (time.Now().Sub(startTime) / time.Second) < initGraceTime {
os.Exit(0)
} else {
endPoint := findEndpoint()
u, err := url.Parse(fmt.Sprintf("http://%s%s", endPoint, healthPath))
if err != nil {
// Could not parse URL successfully
log.Fatal(err.Error())
}
// Minio server may be using self-signed or CA certificates. To avoid
// making Docker setup complicated, we skip verifying certificates here.
// This is because, following request tests for health status within
// containerized environment, i.e. requests are always made to the Minio
// server running on the same host.
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr, Timeout: timeout}
resp, err := client.Get(u.String())
if err != nil {
// GET failed exit
log.Fatal(err.Error())
}
if resp.StatusCode == http.StatusOK {
// Drain any response.
xhttp.DrainBody(resp.Body)
// exit with success
os.Exit(0)
}
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
// Drain any response.
xhttp.DrainBody(resp.Body)
// GET failed exit
log.Fatal(err.Error())
}
bodyString := string(bodyBytes)
// Drain any response.
xhttp.DrainBody(resp.Body)
// This means sever is configured with https
if resp.StatusCode == http.StatusForbidden && bodyString == "SSL required" {
// Try with https
u.Scheme = "https"
resp, err = client.Get(u.String())
if err != nil {
// GET failed exit
log.Fatal(err.Error())
}
if resp.StatusCode == http.StatusOK {
// Drain any response.
xhttp.DrainBody(resp.Body)
// exit with success
os.Exit(0)
}
// Drain any response.
xhttp.DrainBody(resp.Body)
}
}
// Execution reaching here means none of
// the success cases were satisfied
os.Exit(1)
}

56
dockerscripts/healthcheck.sh
View file

@ -1,56 +0,0 @@
#!/bin/sh
#
# Minio Cloud Storage, (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -x
_init () {
scheme="http://"
address="$(netstat -nplt 2>/dev/null | awk ' /(.*\/minio)/ { gsub(":::","127.0.0.1:",$4); print $4}')"
resource="/minio/health/live"
start=$(stat -c "%Y" /proc/1)
}
healthcheck_main () {
# In distributed environment like Swarm, traffic is routed
# to a container only when it reports a `healthy` status. So, we exit
# with 0 to ensure healthy status till distributed Minio starts (120s).
#
# Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272
if [ $(( $(date +%s) - start )) -lt 120 ]; then
exit 0
else
# Get the http response code
http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource})
# Get the http response body
http_response_body=$(curl -k -s ${scheme}${address}${resource})
# server returns response 403 and body "SSL required" if non-TLS
# connection is attempted on a TLS-configured server. Change
# the scheme and try again
if [ "$http_response" = "403" ] && \
[ "$http_response_body" = "SSL required" ]; then
scheme="https://"
http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource})
fi
# If http_response is 200 - server is up.
[ "$http_response" = "200" ]
fi
}
_init && healthcheck_main