Quotas are more accurate, occur on Read()
This commit is contained in:
parent
f94d9db4f7
commit
e2403cbc56
4 changed files with 176 additions and 61 deletions
|
@ -25,6 +25,7 @@ import (
|
||||||
"github.com/minio-io/minio/pkg/api/quota"
|
"github.com/minio-io/minio/pkg/api/quota"
|
||||||
"github.com/minio-io/minio/pkg/iodine"
|
"github.com/minio-io/minio/pkg/iodine"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
"github.com/minio-io/minio/pkg/storage/drivers"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// private use
|
// private use
|
||||||
|
@ -91,6 +92,7 @@ func HTTPHandler(domain string, driver drivers.Driver) http.Handler {
|
||||||
}
|
}
|
||||||
|
|
||||||
h := validateHandler(conf, ignoreResourcesHandler(mux))
|
h := validateHandler(conf, ignoreResourcesHandler(mux))
|
||||||
// quota handler is always last
|
h = quota.BandwidthCap(h, 250*1024*1024, time.Duration(30*time.Minute))
|
||||||
return quota.BandwidthCap(h, int64(100*1024*1024))
|
h = quota.RequestLimit(h, 100, time.Duration(30*time.Minute))
|
||||||
|
return h
|
||||||
}
|
}
|
||||||
|
|
83
pkg/api/quota/bandwidth_cap.go
Normal file
83
pkg/api/quota/bandwidth_cap.go
Normal file
|
@ -0,0 +1,83 @@
|
||||||
|
/*
|
||||||
|
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// bandwidthQuotaHandler
|
||||||
|
type bandwidthQuotaHandler struct {
|
||||||
|
handler http.Handler
|
||||||
|
quotas *quotaMap
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeHTTP is an http.Handler ServeHTTP method
|
||||||
|
func (h *bandwidthQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
|
host, _, _ := net.SplitHostPort(req.RemoteAddr)
|
||||||
|
longIP := longIP{net.ParseIP(host)}.IptoUint32()
|
||||||
|
req.Body = quotaReader{
|
||||||
|
ReadCloser: req.Body,
|
||||||
|
quotas: h.quotas,
|
||||||
|
ip: longIP,
|
||||||
|
}
|
||||||
|
h.handler.ServeHTTP(w, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BandwidthCap sets a quote based upon bandwidth used
|
||||||
|
func BandwidthCap(h http.Handler, limit int64, duration time.Duration) http.Handler {
|
||||||
|
return &bandwidthQuotaHandler{
|
||||||
|
handler: h,
|
||||||
|
quotas: "aMap{
|
||||||
|
data: make(map[int64]map[uint32]int64),
|
||||||
|
limit: int64(limit),
|
||||||
|
duration: duration,
|
||||||
|
segmentSize: segmentSize(duration),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type quotaReader struct {
|
||||||
|
io.ReadCloser
|
||||||
|
quotas *quotaMap
|
||||||
|
ip uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q quotaReader) Read(b []byte) (int, error) {
|
||||||
|
if q.quotas.IsQuotaMet(q.ip) {
|
||||||
|
return 0, errors.New("Quota Met")
|
||||||
|
}
|
||||||
|
n, err := q.ReadCloser.Read(b)
|
||||||
|
q.quotas.Add(q.ip, int64(n))
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q quotaReader) Close() error {
|
||||||
|
return q.ReadCloser.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func segmentSize(duration time.Duration) time.Duration {
|
||||||
|
var segmentSize time.Duration
|
||||||
|
for i := int64(1); i < duration.Nanoseconds(); i = i * 10 {
|
||||||
|
segmentSize = time.Duration(i)
|
||||||
|
}
|
||||||
|
return segmentSize
|
||||||
|
}
|
|
@ -19,7 +19,6 @@ package quota
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
@ -27,38 +26,51 @@ import (
|
||||||
// map[minute][address] = current quota
|
// map[minute][address] = current quota
|
||||||
type quotaMap struct {
|
type quotaMap struct {
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
data map[int64]map[uint32]uint64
|
data map[int64]map[uint32]int64
|
||||||
limit uint64
|
limit int64
|
||||||
duration int64
|
duration time.Duration
|
||||||
|
segmentSize time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func (q *quotaMap) Add(ip uint32, size uint64) bool {
|
func (q *quotaMap) Add(ip uint32, size int64) {
|
||||||
q.Lock()
|
q.Lock()
|
||||||
defer q.Unlock()
|
defer q.Unlock()
|
||||||
currentMinute := time.Now().Unix() / q.duration
|
q.clean()
|
||||||
expiredQuotas := (time.Now().Unix() / q.duration) - 5
|
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
|
||||||
|
if _, ok := q.data[currentMinute]; !ok {
|
||||||
|
q.data[currentMinute] = make(map[uint32]int64)
|
||||||
|
}
|
||||||
|
currentData, _ := q.data[currentMinute][ip]
|
||||||
|
proposedDataSize := currentData + size
|
||||||
|
q.data[currentMinute][ip] = proposedDataSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *quotaMap) IsQuotaMet(ip uint32) bool {
|
||||||
|
q.clean()
|
||||||
|
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
|
||||||
|
if _, ok := q.data[currentMinute]; !ok {
|
||||||
|
q.data[currentMinute] = make(map[uint32]int64)
|
||||||
|
}
|
||||||
|
var total int64
|
||||||
|
for _, segment := range q.data {
|
||||||
|
if used, ok := segment[ip]; ok {
|
||||||
|
total += used
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if total >= q.limit {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *quotaMap) clean() {
|
||||||
|
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
|
||||||
|
expiredQuotas := currentMinute - q.duration.Nanoseconds()
|
||||||
for time := range q.data {
|
for time := range q.data {
|
||||||
if time < expiredQuotas {
|
if time < expiredQuotas {
|
||||||
delete(q.data, time)
|
delete(q.data, time)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, ok := q.data[currentMinute]; !ok {
|
|
||||||
q.data[currentMinute] = make(map[uint32]uint64)
|
|
||||||
}
|
|
||||||
currentData, _ := q.data[currentMinute][ip]
|
|
||||||
proposedDataSize := currentData + size
|
|
||||||
if proposedDataSize > q.limit {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
q.data[currentMinute][ip] = proposedDataSize
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// HttpQuotaHandler
|
|
||||||
type httpQuotaHandler struct {
|
|
||||||
handler http.Handler
|
|
||||||
quotas *quotaMap
|
|
||||||
adder func(uint64) uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type longIP struct {
|
type longIP struct {
|
||||||
|
@ -73,38 +85,3 @@ func (p longIP) IptoUint32() (result uint32) {
|
||||||
}
|
}
|
||||||
return binary.BigEndian.Uint32(ip)
|
return binary.BigEndian.Uint32(ip)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServeHTTP is an http.Handler ServeHTTP method
|
|
||||||
func (h *httpQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|
||||||
host, _, _ := net.SplitHostPort(req.RemoteAddr)
|
|
||||||
longIP := longIP{net.ParseIP(host)}.IptoUint32()
|
|
||||||
if h.quotas.Add(longIP, h.adder(uint64(req.ContentLength))) {
|
|
||||||
h.handler.ServeHTTP(w, req)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// BandwidthCap sets a quote based upon bandwidth used
|
|
||||||
func BandwidthCap(h http.Handler, limit int64) http.Handler {
|
|
||||||
return &httpQuotaHandler{
|
|
||||||
handler: h,
|
|
||||||
quotas: "aMap{
|
|
||||||
data: make(map[int64]map[uint32]uint64),
|
|
||||||
limit: uint64(limit),
|
|
||||||
duration: int64(60),
|
|
||||||
},
|
|
||||||
adder: func(count uint64) uint64 { return count },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RequestLimit sets a quota based upon request count
|
|
||||||
func RequestLimit(h http.Handler, limit int64) http.Handler {
|
|
||||||
return &httpQuotaHandler{
|
|
||||||
handler: h,
|
|
||||||
quotas: "aMap{
|
|
||||||
data: make(map[int64]map[uint32]uint64),
|
|
||||||
limit: uint64(limit),
|
|
||||||
duration: int64(60),
|
|
||||||
},
|
|
||||||
adder: func(count uint64) uint64 { return 1 },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
53
pkg/api/quota/request_limit.go
Normal file
53
pkg/api/quota/request_limit.go
Normal file
|
@ -0,0 +1,53 @@
|
||||||
|
/*
|
||||||
|
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requestLimitHandler
|
||||||
|
type requestLimitHandler struct {
|
||||||
|
handler http.Handler
|
||||||
|
quotas *quotaMap
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeHTTP is an http.Handler ServeHTTP method
|
||||||
|
func (h *requestLimitHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
|
host, _, _ := net.SplitHostPort(req.RemoteAddr)
|
||||||
|
longIP := longIP{net.ParseIP(host)}.IptoUint32()
|
||||||
|
if h.quotas.IsQuotaMet(longIP) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.quotas.Add(longIP, 1)
|
||||||
|
h.handler.ServeHTTP(w, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestLimit sets a quote based upon number of requests allowed over a time period
|
||||||
|
func RequestLimit(h http.Handler, limit int64, duration time.Duration) http.Handler {
|
||||||
|
return &requestLimitHandler{
|
||||||
|
handler: h,
|
||||||
|
quotas: "aMap{
|
||||||
|
data: make(map[int64]map[uint32]int64),
|
||||||
|
limit: int64(limit),
|
||||||
|
duration: duration,
|
||||||
|
segmentSize: segmentSize(duration),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue