maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix k8s replica set deployment (#8629)

Browse source 
In replica sets, hosts resolve to localhost
IP automatically until the deployment fully
comes up. To avoid this issue we need to
wait for such resolution.
This commit is contained in:
Harshavardhana 2019-12-10 20:28:22 -08:00 committed by GitHub
parent 46b4dd8e20
commit 3e9ab5f4a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 189 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.travis.yml
View file

@ -26,6 +26,7 @@ matrix:
- ARCH=x86_64
- CGO_ENABLED=0
- GO111MODULE=on
- SIMPLE_CI=1
go: 1.13.x
script:
- make
@ -43,6 +44,7 @@ matrix:
- ARCH=x86_64
- CGO_ENABLED=0
- GO111MODULE=on
- SIMPLE_CI=1
go: 1.13.x
script:
- go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe

4
Dockerfile.simpleci
View file

@ -8,6 +8,7 @@ WORKDIR /go/src/github.com/minio/minio
RUN apt-get update && apt-get install -y jq
ENV GO111MODULE=on
ENV SIMPLE_CI 1
RUN git config --global http.cookiefile /gitcookie/.gitcookie
@ -43,6 +44,8 @@ RUN make verify
#-------------------------------------------------------------
FROM node:10.15-stretch-slim
ENV SIMPLE_CI 1
COPY browser /minio/browser
WORKDIR /minio/browser
@ -63,6 +66,7 @@ ENV LANG C.UTF-8
ENV GOROOT /usr/local/go
ENV GOPATH /usr/local
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
ENV SIMPLE_CI 1
ENV MINT_ROOT_DIR /mint
RUN apt-get --yes update && apt-get --yes upgrade && \

97
cmd/endpoint.go
View file

@ -18,6 +18,7 @@ package cmd
import (
"context"
"errors"
"fmt"
"net"
"net/url"
@ -229,8 +230,46 @@ func (endpoints Endpoints) GetString(i int) string {
return endpoints[i].String()
}
func (endpoints Endpoints) atleastOneEndpointLocal() bool {
for _, endpoint := range endpoints {
if endpoint.IsLocal {
return true
}
}
return false
}
func (endpoints Endpoints) doAllHostsResolveToLocalhost() bool {
var endpointHosts = map[string]set.StringSet{}
for _, endpoint := range endpoints {
hostIPs, err := getHostIP(endpoint.Hostname())
if err != nil {
continue
}
endpointHosts[endpoint.Hostname()] = hostIPs
}
sameHosts := make(map[string]int)
for hostName, endpointIPs := range endpointHosts {
for _, endpointIP := range endpointIPs.ToSlice() {
if net.ParseIP(endpointIP).IsLoopback() {
sameHosts[hostName]++
}
}
}
ok := true
for _, localCount := range sameHosts {
ok = ok && localCount > 0
}
if len(sameHosts) == 0 {
return false
}
return ok
}
// UpdateIsLocal - resolves the host and discovers the local host.
func (endpoints Endpoints) UpdateIsLocal() error {
orchestrated := IsDocker() || IsKubernetes()
var epsResolved int
var foundLocal bool
resolvedList := make([]bool, len(endpoints))
@ -256,29 +295,61 @@ func (endpoints Endpoints) UpdateIsLocal() error {
continue
}
// return err if not Docker or Kubernetes
// We use IsDocker() to check for Docker environment
// We use IsKubernetes() to check for Kubernetes environment
isLocal, err := isLocalHost(endpoints[i].Hostname(), endpoints[i].Port(), globalMinioPort)
if err != nil {
if !IsDocker() && !IsKubernetes() {
return err
}
// Log the message to console about the host resolving
reqInfo := (&logger.ReqInfo{}).AppendTags(
"host",
endpoints[i].Hostname(),
)
if orchestrated && endpoints.doAllHostsResolveToLocalhost() {
err := errors.New("hosts resolve to same IP, DNS not updated on k8s")
// time elapsed
timeElapsed := time.Since(startTime)
// log error only if more than 1s elapsed
if timeElapsed > time.Second {
// Log the message to console about the host not being resolveable.
reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].Hostname())
reqInfo.AppendTags("elapsedTime",
humanize.RelTime(startTime, startTime.Add(timeElapsed),
"elapsed", ""))
humanize.RelTime(startTime,
startTime.Add(timeElapsed),
"elapsed",
""))
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err, logger.Application)
}
continue
}
// return err if not Docker or Kubernetes
// We use IsDocker() to check for Docker environment
// We use IsKubernetes() to check for Kubernetes environment
isLocal, err := isLocalHost(endpoints[i].Hostname(),
endpoints[i].Port(),
globalMinioPort,
)
if err != nil && !orchestrated {
return err
}
if err != nil {
// time elapsed
timeElapsed := time.Since(startTime)
// log error only if more than 1s elapsed
if timeElapsed > time.Second {
reqInfo.AppendTags("elapsedTime",
humanize.RelTime(startTime,
startTime.Add(timeElapsed),
"elapsed",
"",
))
ctx := logger.SetReqInfo(context.Background(),
reqInfo)
logger.LogIf(ctx, err, logger.Application)
}
} else {
resolvedList[i] = true
endpoints[i].IsLocal = isLocal
if orchestrated && !endpoints.atleastOneEndpointLocal() {
resolvedList[i] = false
continue
}
epsResolved++
if !foundLocal {
foundLocal = isLocal
@ -288,7 +359,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
// Wait for the tick, if the there exist a local endpoint in discovery.
// Non docker/kubernetes environment we do not need to wait.
if !foundLocal && (IsDocker() || IsKubernetes()) {
if !foundLocal && orchestrated {
<-keepAliveTicker.C
}
}

96
cmd/iam.go
View file

@ -221,13 +221,13 @@ type IAMStorageAPI interface {
// simplifies the implementation for group removal. This is called
// only via IAM notifications.
func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
if objAPI == nil {
return errInvalidArgument
}
sys.Lock()
defer sys.Unlock()
if objAPI == nil || sys.store == nil {
return errServerNotInitialized
}
if globalEtcdClient != nil {
// Watch APIs cover this case, so nothing to do.
return nil
@ -262,13 +262,13 @@ func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
// LoadPolicy - reloads a specific canned policy from backend disks or etcd.
func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
if objAPI == nil {
return errInvalidArgument
}
sys.Lock()
defer sys.Unlock()
if objAPI == nil || sys.store == nil {
return errServerNotInitialized
}
if globalEtcdClient == nil {
return sys.store.loadPolicyDoc(policyName, sys.iamPolicyDocsMap)
}
@ -280,13 +280,13 @@ func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
// LoadPolicyMapping - loads the mapped policy for a user or group
// from storage into server memory.
func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isGroup bool) error {
if objAPI == nil {
return errInvalidArgument
}
sys.Lock()
defer sys.Unlock()
if objAPI == nil || sys.store == nil {
return errServerNotInitialized
}
if globalEtcdClient == nil {
var err error
if isGroup {
@ -306,13 +306,13 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
// LoadUser - reloads a specific user from backend disks or etcd.
func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, isSTS bool) error {
if objAPI == nil {
return errInvalidArgument
}
sys.Lock()
defer sys.Unlock()
if objAPI == nil || sys.store == nil {
return errServerNotInitialized
}
if globalEtcdClient == nil {
err := sys.store.loadUser(accessKey, isSTS, sys.iamUsersMap)
if err != nil {
@ -351,14 +351,16 @@ func (sys *IAMSys) doIAMConfigMigration(objAPI ObjectLayer) error {
// Init - initializes config system from iam.json
func (sys *IAMSys) Init(objAPI ObjectLayer) error {
if objAPI == nil {
return errInvalidArgument
return errServerNotInitialized
}
sys.Lock()
if globalEtcdClient == nil {
sys.store = newIAMObjectStore()
} else {
sys.store = newIAMEtcdStore()
}
sys.Unlock()
doneCh := make(chan struct{})
defer close(doneCh)
@ -416,6 +418,13 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
return errInvalidArgument
}
sys.Lock()
defer sys.Unlock()
if sys.store == nil {
return errServerNotInitialized
}
err := sys.store.deletePolicyDoc(policyName)
switch err.(type) {
case ObjectNotFound:
@ -423,9 +432,6 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
err = nil
}
sys.Lock()
defer sys.Unlock()
delete(sys.iamPolicyDocsMap, policyName)
return err
}
@ -481,12 +487,17 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
return errInvalidArgument
}
sys.Lock()
defer sys.Unlock()
if sys.store == nil {
return errServerNotInitialized
}
if err := sys.store.savePolicyDoc(policyName, p); err != nil {
return err
}
sys.Lock()
defer sys.Unlock()
sys.iamPolicyDocsMap[policyName] = p
return nil
}
@ -505,6 +516,10 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
return errIAMActionNotAllowed
}
if sys.store == nil {
return errServerNotInitialized
}
// It is ok to ignore deletion error on the mapped policy
sys.store.deleteMappedPolicy(accessKey, false, false)
err := sys.store.deleteUserIdentity(accessKey, false)
@ -543,6 +558,10 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
return nil
}
if sys.store == nil {
return errServerNotInitialized
}
mp := newMappedPolicy(policyName)
if err := sys.store.saveMappedPolicy(accessKey, true, false, mp); err != nil {
return err
@ -551,6 +570,10 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
sys.iamUserPolicyMap[accessKey] = mp
}
if sys.store == nil {
return errServerNotInitialized
}
u := newUserIdentity(cred)
if err := sys.store.saveUserIdentity(accessKey, true, u); err != nil {
return err
@ -659,6 +682,11 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
return config.EnableOff
}(),
})
if sys.store == nil {
return errServerNotInitialized
}
if err := sys.store.saveUserIdentity(accessKey, false, uinfo); err != nil {
return err
}
@ -687,6 +715,10 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
return errIAMActionNotAllowed
}
if sys.store == nil {
return errServerNotInitialized
}
if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
return err
}
@ -718,6 +750,10 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
return errNoSuchUser
}
if sys.store == nil {
return errServerNotInitialized
}
cred.SecretKey = secretKey
u := newUserIdentity(cred)
if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
@ -775,6 +811,10 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
gi.Members = uniqMembers
}
if sys.store == nil {
return errServerNotInitialized
}
if err := sys.store.saveGroupInfo(group, gi); err != nil {
return err
}
@ -832,6 +872,10 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
return errGroupNotEmpty
}
if sys.store == nil {
return errServerNotInitialized
}
if len(members) == 0 {
// len(gi.Members) == 0 here.
@ -887,6 +931,10 @@ func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
sys.Lock()
defer sys.Unlock()
if sys.store == nil {
return errServerNotInitialized
}
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
@ -984,6 +1032,10 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
// policyDBSet - sets a policy for user in the policy db. Assumes that
// caller has sys.Lock().
func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
if sys.store == nil {
return errServerNotInitialized
}
if name == "" || policy == "" {
return errInvalidArgument
}

41
cmd/update.go
View file

@ -130,31 +130,40 @@ func GetCurrentReleaseTime() (releaseTime time.Time, err error) {
// "/.dockerenv": "file",
//
func IsDocker() bool {
_, err := os.Stat("/.dockerenv")
if os.IsNotExist(err) {
return false
if env.Get("SIMPLE_CI", "") == "" {
_, err := os.Stat("/.dockerenv")
if os.IsNotExist(err) {
return false
}
// Log error, as we will not propagate it to caller
logger.LogIf(context.Background(), err)
return err == nil
}
// Log error, as we will not propagate it to caller
logger.LogIf(context.Background(), err)
return err == nil
return false
}
// IsDCOS returns true if minio is running in DCOS.
func IsDCOS() bool {
// http://mesos.apache.org/documentation/latest/docker-containerizer/
// Mesos docker containerizer sets this value
return env.Get("MESOS_CONTAINER_NAME", "") != ""
if env.Get("SIMPLE_CI", "") == "" {
// http://mesos.apache.org/documentation/latest/docker-containerizer/
// Mesos docker containerizer sets this value
return env.Get("MESOS_CONTAINER_NAME", "") != ""
}
return false
}
// IsKubernetes returns true if minio is running in kubernetes.
func IsKubernetes() bool {
// Kubernetes env used to validate if we are
// indeed running inside a kubernetes pod
// is KUBERNETES_SERVICE_HOST but in future
// we might need to enhance this.
return env.Get("KUBERNETES_SERVICE_HOST", "") != ""
if env.Get("SIMPLE_CI", "") == "" {
// Kubernetes env used to validate if we are
// indeed running inside a kubernetes pod
// is KUBERNETES_SERVICE_HOST but in future
// we might need to enhance this.
return env.Get("KUBERNETES_SERVICE_HOST", "") != ""
}
return false
}
// IsBOSH returns true if minio is deployed from a bosh package