/* * Minio Cloud Storage, (C) 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package cmd import ( "context" "fmt" "io/ioutil" "time" "github.com/Sirupsen/logrus" "gopkg.in/olivere/elastic.v5" ) var ( esErrFunc = newNotificationErrorFactory("Elasticsearch") errESFormat = esErrFunc(`"format" value is invalid - it must be one of "%s" or "%s".`, formatNamespace, formatAccess) errESIndex = esErrFunc("Index name was not specified in the configuration.") ) // elasticQueue is a elasticsearch event notification queue. type elasticSearchNotify struct { Enable bool `json:"enable"` Format string `json:"format"` URL string `json:"url"` Index string `json:"index"` } func (e *elasticSearchNotify) Validate() error { if !e.Enable { return nil } if e.Format != formatNamespace && e.Format != formatAccess { return errESFormat } if _, err := checkURL(e.URL); err != nil { return err } if e.Index == "" { return errESIndex } return nil } type elasticClient struct { *elastic.Client params elasticSearchNotify } // Connects to elastic search instance at URL. func dialElastic(esNotify elasticSearchNotify) (*elastic.Client, error) { if !esNotify.Enable { return nil, errNotifyNotEnabled } return elastic.NewClient( elastic.SetURL(esNotify.URL), elastic.SetSniff(false), elastic.SetMaxRetries(10), ) } func newElasticNotify(accountID string) (*logrus.Logger, error) { esNotify := serverConfig.Notify.GetElasticSearchByID(accountID) // Dial to elastic search. client, err := dialElastic(esNotify) if err != nil { return nil, esErrFunc("Error dialing the server: %v", err) } // Use the IndexExists service to check if a specified index exists. exists, err := client.IndexExists(esNotify.Index). Do(context.Background()) if err != nil { return nil, esErrFunc("Error checking if index exists: %v", err) } // Index does not exist, attempt to create it. if !exists { var createIndex *elastic.IndicesCreateResult createIndex, err = client.CreateIndex(esNotify.Index). Do(context.Background()) if err != nil { return nil, esErrFunc("Error creating index `%s`: %v", esNotify.Index, err) } if !createIndex.Acknowledged { return nil, esErrFunc("Index not created") } } elasticCl := elasticClient{ Client: client, params: esNotify, } elasticSearchLog := logrus.New() // Disable writing to console. elasticSearchLog.Out = ioutil.Discard // Add a elasticSearch hook. elasticSearchLog.Hooks.Add(elasticCl) // Set default JSON formatter. elasticSearchLog.Formatter = new(logrus.JSONFormatter) // Success, elastic search successfully initialized. return elasticSearchLog, nil } // Fire is required to implement logrus hook func (q elasticClient) Fire(entry *logrus.Entry) (err error) { // Reflect on eventType and Key on their native type. entryStr, ok := entry.Data["EventType"].(string) if !ok { return nil } keyStr, ok := entry.Data["Key"].(string) if !ok { return nil } switch q.params.Format { case formatNamespace: // If event matches as delete, we purge the previous index. if eventMatch(entryStr, []string{"s3:ObjectRemoved:*"}) { _, err = q.Client.Delete().Index(q.params.Index). Type("event").Id(keyStr).Do(context.Background()) break } // else we update elastic index or create a new one. _, err = q.Client.Index().Index(q.params.Index). Type("event"). BodyJson(map[string]interface{}{ "Records": entry.Data["Records"], }).Id(keyStr).Do(context.Background()) case formatAccess: // eventTime is taken from the first entry in the // records. events, ok := entry.Data["Records"].([]NotificationEvent) if !ok { return esErrFunc("Unable to extract event time due to conversion error of entry.Data[\"Records\"]=%v", entry.Data["Records"]) } var eventTime time.Time eventTime, err = time.Parse(timeFormatAMZ, events[0].EventTime) if err != nil { return esErrFunc("Unable to parse event time \"%s\": %v", events[0].EventTime, err) } // Extract event time in milliseconds for Elasticsearch. eventTimeStr := fmt.Sprintf("%d", eventTime.UnixNano()/1000000) _, err = q.Client.Index().Index(q.params.Index).Type("event"). Timestamp(eventTimeStr). BodyJson(map[string]interface{}{ "Records": entry.Data["Records"], }).Do(context.Background()) } if err != nil { return esErrFunc("Error inserting/deleting entry: %v", err) } return nil } // Required for logrus hook implementation func (q elasticClient) Levels() []logrus.Level { return []logrus.Level{ logrus.InfoLevel, } }