feat: time to bring back http2.0 support (#10230)
Bonus move our CI/CD to go1.14
This commit is contained in:
Harshavardhana
GitHub
parent
a49e3647b6
commit
1e2ebc9945
2
.github/workflows/go.yml
vendored
2
.github/workflows/go.yml
vendored
|
|
@ -12,7 +12,7 @@ jobs:
|
|||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.13.x]
|
||||
go-version: [1.14.x]
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
|
|||
|
||||
rq := trace.RequestInfo{
|
||||
Time: time.Now().UTC(),
|
||||
Proto: r.Proto,
|
||||
Method: r.Method,
|
||||
Path: r.URL.Path,
|
||||
RawQuery: r.URL.RawQuery,
|
||||
|
|
|
|||
|
|
@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
|
|||
// TLS hardening
|
||||
PreferServerCipherSuites: true,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
// Do not edit the next line, protos priority is kept
|
||||
// on purpose in this manner for HTTP 2.0, we would
|
||||
// still like HTTP 2.0 clients to negotiate connection
|
||||
// to server if needed but by default HTTP 1.1 is
|
||||
// expected. We need to change this in future
|
||||
// when we wish to go back to HTTP 2.0 as default
|
||||
// priority for HTTP protocol negotiation.
|
||||
NextProtos: []string{"http/1.1", "h2"},
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
}
|
||||
tlsConfig.GetCertificate = getCert
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
|
|||
tlsConfig = &tls.Config{
|
||||
ServerName: endpoint.Hostname(),
|
||||
RootCAs: globalRootCAs,
|
||||
NextProtos: []string{"http/1.1"}, // Force http1.1
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error {
|
|||
tlsConfig = &tls.Config{
|
||||
ServerName: endpoint.Hostname(),
|
||||
RootCAs: globalRootCAs,
|
||||
NextProtos: []string{"http/1.1"}, // Force http1.1
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ package cmd
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
|
|
@ -73,7 +75,7 @@ func handleSignals() {
|
|||
if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
|
||||
objAPI.Shutdown(context.Background())
|
||||
}
|
||||
if err != nil {
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
logger.Fatal(err, "Unable to start MinIO server")
|
||||
}
|
||||
exit(true)
|
||||
|
|
|
|||
|
|
@ -663,7 +663,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
|
|||
tlsConfig = &tls.Config{
|
||||
ServerName: endpoint.Hostname(),
|
||||
RootCAs: globalRootCAs,
|
||||
NextProtos: []string{"http/1.1"}, // Force http1.1
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
16
cmd/utils.go
16
cmd/utils.go
|
|
@ -39,13 +39,13 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gorilla/mux"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gorilla/mux"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -467,6 +467,11 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
|
|||
// in raw stream.
|
||||
DisableCompression: true,
|
||||
}
|
||||
|
||||
if tlsConfig != nil {
|
||||
http2.ConfigureTransport(tr)
|
||||
}
|
||||
|
||||
return func() *http.Transport {
|
||||
return tr
|
||||
}
|
||||
|
|
@ -490,6 +495,11 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
|
|||
// in raw stream.
|
||||
DisableCompression: true,
|
||||
}
|
||||
|
||||
if tlsConfig != nil {
|
||||
http2.ConfigureTransport(tr)
|
||||
}
|
||||
|
||||
return func() *http.Transport {
|
||||
return tr
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -50,7 +50,7 @@ require (
|
|||
github.com/minio/minio-go/v7 v7.0.3
|
||||
github.com/minio/selfupdate v0.3.1
|
||||
github.com/minio/sha256-simd v0.1.1
|
||||
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37
|
||||
github.com/minio/simdjson-go v0.1.5
|
||||
github.com/minio/sio v0.2.0
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -297,8 +297,8 @@ github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlE
|
|||
github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
|
||||
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 h1:pDeao6M5AEd8hwTtGmE0pVKomlL56JFRa5SiXDZAuJE=
|
||||
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
|
||||
github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1kM=
|
||||
github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
|
||||
github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
|
||||
github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
|
|
|
|||
|
|
@ -15,37 +15,42 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import io
|
||||
import os
|
||||
|
||||
from minio import Minio
|
||||
from minio.select.options import (SelectObjectOptions, CSVInput,
|
||||
RequestProgress, InputSerialization,
|
||||
OutputSerialization, CSVOutput, JsonOutput)
|
||||
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
|
||||
JsonOutput, OutputSerialization,
|
||||
RequestProgress, SelectObjectOptions)
|
||||
|
||||
from utils import *
|
||||
|
||||
|
||||
def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output):
|
||||
""" Test if the passed SQL request has the output equal to the passed execpted one"""
|
||||
object_name = generate_object_name()
|
||||
got_output = b''
|
||||
try:
|
||||
bytes_content = io.BytesIO(input_data)
|
||||
client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data))
|
||||
client.put_object(bucket_name, object_name,
|
||||
io.BytesIO(input_data), len(input_data))
|
||||
data = client.select_object_content(bucket_name, object_name, sql_opts)
|
||||
# Get the records
|
||||
records = io.BytesIO()
|
||||
for d in data.stream(10*1024):
|
||||
records.write(d.encode('utf-8'))
|
||||
got_output = records.getvalue()
|
||||
got_output = records.getvalue()
|
||||
except Exception as select_err:
|
||||
if not isinstance(expected_output, Exception):
|
||||
raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
|
||||
raise ValueError(
|
||||
'Test {} unexpectedly failed with: {}'.format(test_name, select_err))
|
||||
else:
|
||||
if isinstance(expected_output, Exception):
|
||||
raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output))
|
||||
raise ValueError(
|
||||
'Test {}: expected an exception, got {}'.format(test_name, got_output))
|
||||
if got_output != expected_output:
|
||||
raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output))
|
||||
raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(
|
||||
test_name, expected_output, got_output))
|
||||
finally:
|
||||
client.remove_object(bucket_name, object_name)
|
||||
|
||||
|
|
@ -55,28 +60,34 @@ def test_csv_input_custom_quote_char(client, log_output):
|
|||
log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
|
||||
|
||||
tests = [
|
||||
# Invalid quote character, should fail
|
||||
('""', '"', b'col1,col2,col3\n', Exception()),
|
||||
# UTF-8 quote character
|
||||
('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
|
||||
# Only one field is quoted
|
||||
('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
|
||||
('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
|
||||
('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
|
||||
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
|
||||
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
|
||||
('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
|
||||
('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
|
||||
('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
|
||||
('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
|
||||
('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'),
|
||||
('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'),
|
||||
('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'),
|
||||
('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'),
|
||||
# Invalid quote character, should fail
|
||||
('""', '"', b'col1,col2,col3\n', Exception()),
|
||||
# UTF-8 quote character
|
||||
('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(),
|
||||
b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
|
||||
# Only one field is quoted
|
||||
('"', '"', b'"col1",col2,col3\n',
|
||||
b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
|
||||
('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
|
||||
('\'', '"', b'"col1",col2,col3\n',
|
||||
b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
|
||||
('', '"', b'"col1",col2,col3\n',
|
||||
b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
|
||||
('', '"', b'"col1",col2,col3\n',
|
||||
b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
|
||||
('', '"', b'"col1","col2","col3"\n',
|
||||
b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
|
||||
('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
|
||||
('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
|
||||
('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
|
||||
('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'),
|
||||
('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
|
||||
('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'),
|
||||
('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'),
|
||||
('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'),
|
||||
]
|
||||
|
||||
client.make_bucket(bucket_name)
|
||||
|
|
@ -84,54 +95,56 @@ def test_csv_input_custom_quote_char(client, log_output):
|
|||
try:
|
||||
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
|
||||
sql_opts = SelectObjectOptions(
|
||||
expression="select * from s3object",
|
||||
input_serialization=InputSerialization(
|
||||
compression_type="NONE",
|
||||
csv=CSVInput(FileHeaderInfo="NONE",
|
||||
RecordDelimiter="\n",
|
||||
FieldDelimiter=",",
|
||||
QuoteCharacter=quote_char,
|
||||
QuoteEscapeCharacter=escape_char,
|
||||
Comments="#",
|
||||
AllowQuotedRecordDelimiter="FALSE",),
|
||||
),
|
||||
output_serialization=OutputSerialization(
|
||||
json = JsonOutput(
|
||||
RecordDelimiter="\n",
|
||||
)
|
||||
),
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
expression="select * from s3object",
|
||||
input_serialization=InputSerialization(
|
||||
compression_type="NONE",
|
||||
csv=CSVInput(FileHeaderInfo="NONE",
|
||||
RecordDelimiter="\n",
|
||||
FieldDelimiter=",",
|
||||
QuoteCharacter=quote_char,
|
||||
QuoteEscapeCharacter=escape_char,
|
||||
Comments="#",
|
||||
AllowQuotedRecordDelimiter="FALSE",),
|
||||
),
|
||||
output_serialization=OutputSerialization(
|
||||
json=JsonOutput(
|
||||
RecordDelimiter="\n",
|
||||
)
|
||||
),
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
)
|
||||
|
||||
test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output)
|
||||
test_sql_api(f'test_{idx}', client, bucket_name,
|
||||
data, sql_opts, expected_output)
|
||||
finally:
|
||||
client.remove_bucket(bucket_name)
|
||||
|
||||
# Test passes
|
||||
print(log_output.json_report())
|
||||
|
||||
|
||||
def test_csv_output_custom_quote_char(client, log_output):
|
||||
# Get a unique bucket_name and object_name
|
||||
log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
|
||||
|
||||
tests = [
|
||||
# UTF-8 quote character
|
||||
("''", "''", b'col1,col2,col3\n', Exception()),
|
||||
("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
|
||||
("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'),
|
||||
('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'),
|
||||
('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'),
|
||||
('"', '"', b'""""\n', b'""""\n'),
|
||||
('"', '"', b'\n', b''),
|
||||
("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
|
||||
("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"),
|
||||
("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"),
|
||||
("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"),
|
||||
("'", "\\", b'col\'\n', b"'col\\''\n"),
|
||||
# Two consecutive escaped quotes
|
||||
("'", "\\", b'"a"""""\n', b"'a\"\"'\n"),
|
||||
# UTF-8 quote character
|
||||
("''", "''", b'col1,col2,col3\n', Exception()),
|
||||
("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
|
||||
("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'),
|
||||
('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'),
|
||||
('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'),
|
||||
('"', '"', b'""""\n', b'""""\n'),
|
||||
('"', '"', b'\n', b''),
|
||||
("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
|
||||
("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"),
|
||||
("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"),
|
||||
("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"),
|
||||
("'", "\\", b'col\'\n', b"'col\\''\n"),
|
||||
# Two consecutive escaped quotes
|
||||
("'", "\\", b'"a"""""\n', b"'a\"\"'\n"),
|
||||
]
|
||||
|
||||
client.make_bucket(bucket_name)
|
||||
|
|
@ -139,34 +152,33 @@ def test_csv_output_custom_quote_char(client, log_output):
|
|||
try:
|
||||
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
|
||||
sql_opts = SelectObjectOptions(
|
||||
expression="select * from s3object",
|
||||
input_serialization=InputSerialization(
|
||||
compression_type="NONE",
|
||||
csv=CSVInput(FileHeaderInfo="NONE",
|
||||
RecordDelimiter="\n",
|
||||
FieldDelimiter=",",
|
||||
QuoteCharacter='"',
|
||||
QuoteEscapeCharacter='"',
|
||||
Comments="#",
|
||||
AllowQuotedRecordDelimiter="FALSE",),
|
||||
),
|
||||
output_serialization=OutputSerialization(
|
||||
csv=CSVOutput(QuoteFields="ALWAYS",
|
||||
RecordDelimiter="\n",
|
||||
FieldDelimiter=",",
|
||||
QuoteCharacter=quote_char,
|
||||
QuoteEscapeCharacter=escape_char,)
|
||||
),
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
)
|
||||
expression="select * from s3object",
|
||||
input_serialization=InputSerialization(
|
||||
compression_type="NONE",
|
||||
csv=CSVInput(FileHeaderInfo="NONE",
|
||||
RecordDelimiter="\n",
|
||||
FieldDelimiter=",",
|
||||
QuoteCharacter='"',
|
||||
QuoteEscapeCharacter='"',
|
||||
Comments="#",
|
||||
AllowQuotedRecordDelimiter="FALSE",),
|
||||
),
|
||||
output_serialization=OutputSerialization(
|
||||
csv=CSVOutput(QuoteFields="ALWAYS",
|
||||
RecordDelimiter="\n",
|
||||
FieldDelimiter=",",
|
||||
QuoteCharacter=quote_char,
|
||||
QuoteEscapeCharacter=escape_char,)
|
||||
),
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
)
|
||||
|
||||
test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output)
|
||||
test_sql_api(f'test_{idx}', client, bucket_name,
|
||||
input_data, sql_opts, expected_output)
|
||||
finally:
|
||||
client.remove_bucket(bucket_name)
|
||||
|
||||
# Test passes
|
||||
print(log_output.json_report())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,14 +18,15 @@
|
|||
import io
|
||||
from datetime import datetime
|
||||
|
||||
from minio import Minio
|
||||
from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput,
|
||||
RequestProgress, InputSerialization,
|
||||
OutputSerialization, CSVOutput, JsonOutput)
|
||||
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
|
||||
JSONInput, OutputSerialization,
|
||||
RequestProgress, SelectObjectOptions)
|
||||
|
||||
from utils import *
|
||||
from utils import generate_bucket_name, generate_object_name
|
||||
|
||||
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output):
|
||||
|
||||
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
|
||||
sql_output, tests, log_output):
|
||||
bucket_name = generate_bucket_name()
|
||||
object_name = generate_object_name()
|
||||
|
||||
|
|
@ -48,10 +49,11 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
|
|||
output_serialization=sql_output,
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
data = client.select_object_content(bucket_name, object_name, options)
|
||||
data = client.select_object_content(
|
||||
bucket_name, object_name, options)
|
||||
|
||||
# Get the records
|
||||
records = io.BytesIO()
|
||||
|
|
@ -62,13 +64,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
|
|||
if got_output != expected_output:
|
||||
if type(expected_output) == datetime:
|
||||
# Attempt to parse the date which will throw an exception for any issue
|
||||
datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
|
||||
datetime.strptime(got_output.decode(
|
||||
"utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
|
||||
else:
|
||||
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output))
|
||||
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(
|
||||
idx+1, expected_output, got_output))
|
||||
|
||||
log_output.args['total_success'] += 1
|
||||
except Exception as err:
|
||||
continue ## TODO, raise instead
|
||||
continue # TODO, raise instead
|
||||
# raise Exception(err)
|
||||
finally:
|
||||
client.remove_object(bucket_name, object_name)
|
||||
|
|
@ -77,16 +81,16 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
|
|||
|
||||
def test_sql_expressions(client, input_json_bytes, tests, log_output):
|
||||
input_serialization = InputSerialization(
|
||||
compression_type="NONE",
|
||||
json=JSONInput(Type="DOCUMENT"),
|
||||
)
|
||||
compression_type="NONE",
|
||||
json=JSONInput(Type="DOCUMENT"),
|
||||
)
|
||||
|
||||
output_serialization=OutputSerialization(
|
||||
csv=CSVOutput(QuoteFields="ASNEEDED")
|
||||
)
|
||||
output_serialization = OutputSerialization(
|
||||
csv=CSVOutput(QuoteFields="ASNEEDED")
|
||||
)
|
||||
|
||||
test_sql_expressions_custom_input_output(client, input_json_bytes,
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
|
||||
|
||||
def test_sql_operators(client, log_output):
|
||||
|
|
@ -98,31 +102,38 @@ def test_sql_operators(client, log_output):
|
|||
"""
|
||||
|
||||
tests = [
|
||||
# Logical operators
|
||||
("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
|
||||
("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
|
||||
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'),
|
||||
# Comparison Operators
|
||||
("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
|
||||
(">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
|
||||
("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'),
|
||||
(">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
|
||||
("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
|
||||
("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
|
||||
("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
|
||||
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'),
|
||||
("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
|
||||
# Pattern Matching Operators
|
||||
("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
|
||||
("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
|
||||
# Unitary Operators
|
||||
("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
|
||||
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
|
||||
# Math Operators
|
||||
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
|
||||
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
|
||||
("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'),
|
||||
("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'),
|
||||
# Logical operators
|
||||
("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
|
||||
("NOT", "select * from S3Object s where NOT s.id = 1",
|
||||
b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
|
||||
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3",
|
||||
b'1,John,3\n3,Yves,5\n'),
|
||||
# Comparison Operators
|
||||
("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
|
||||
(">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
|
||||
("<=", "select * from S3Object s where s.age <= 4",
|
||||
b'1,John,3\n2,Elliot,4\n4,,0\n'),
|
||||
(">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
|
||||
("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
|
||||
("<>", "select * from S3Object s where s.age <> 4",
|
||||
b'1,John,3\n3,Yves,5\n4,,0\n'),
|
||||
("!=", "select * from S3Object s where s.age != 4",
|
||||
b'1,John,3\n3,Yves,5\n4,,0\n'),
|
||||
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5",
|
||||
b'2,Elliot,4\n3,Yves,5\n'),
|
||||
("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
|
||||
# Pattern Matching Operators
|
||||
("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
|
||||
("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
|
||||
# Unitary Operators
|
||||
("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
|
||||
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL",
|
||||
b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
|
||||
# Math Operators
|
||||
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
|
||||
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
|
||||
("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'),
|
||||
("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'),
|
||||
]
|
||||
|
||||
try:
|
||||
|
|
@ -141,20 +152,20 @@ def test_sql_operators_precedence(client, log_output):
|
|||
json_testfile = """{"id": 1, "name": "Eric"}"""
|
||||
|
||||
tests = [
|
||||
("-_1", "select -3*3 from S3Object", b'-9\n'),
|
||||
("*", "select 10-3*2 from S3Object", b'4\n'),
|
||||
("/", "select 13-10/5 from S3Object", b'11\n'),
|
||||
("%", "select 13-10%5 from S3Object", b'13\n'),
|
||||
("+", "select 1+1*3 from S3Object", b'4\n'),
|
||||
("-_2", "select 1-1*3 from S3Object", b'-2\n'),
|
||||
("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'),
|
||||
("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'),
|
||||
("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
|
||||
("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'),
|
||||
("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
|
||||
("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'),
|
||||
("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'),
|
||||
("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'),
|
||||
("-_1", "select -3*3 from S3Object", b'-9\n'),
|
||||
("*", "select 10-3*2 from S3Object", b'4\n'),
|
||||
("/", "select 13-10/5 from S3Object", b'11\n'),
|
||||
("%", "select 13-10%5 from S3Object", b'13\n'),
|
||||
("+", "select 1+1*3 from S3Object", b'4\n'),
|
||||
("-_2", "select 1-1*3 from S3Object", b'-2\n'),
|
||||
("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'),
|
||||
("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'),
|
||||
("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
|
||||
("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'),
|
||||
("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
|
||||
("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'),
|
||||
("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'),
|
||||
("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'),
|
||||
]
|
||||
|
||||
try:
|
||||
|
|
@ -168,7 +179,6 @@ def test_sql_operators_precedence(client, log_output):
|
|||
print(log_output.json_report())
|
||||
|
||||
|
||||
|
||||
def test_sql_functions_agg_cond_conv(client, log_output):
|
||||
|
||||
json_testfile = """{"id": 1, "name": "John", "age": 3}
|
||||
|
|
@ -178,17 +188,18 @@ def test_sql_functions_agg_cond_conv(client, log_output):
|
|||
{"id": 5, "name": "Eric", "age": 0}
|
||||
"""
|
||||
tests = [
|
||||
# Aggregate functions
|
||||
("COUNT", "select count(*) from S3Object s", b'5\n'),
|
||||
("AVG", "select avg(s.age) from S3Object s", b'3\n'),
|
||||
("MAX", "select max(s.age) from S3Object s", b'5\n'),
|
||||
("MIN", "select min(s.age) from S3Object s", b'0\n'),
|
||||
("SUM", "select sum(s.age) from S3Object s", b'12\n'),
|
||||
# Conditional functions
|
||||
("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
|
||||
("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
|
||||
## Conversion functions
|
||||
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'),
|
||||
# Aggregate functions
|
||||
("COUNT", "select count(*) from S3Object s", b'5\n'),
|
||||
("AVG", "select avg(s.age) from S3Object s", b'3\n'),
|
||||
("MAX", "select max(s.age) from S3Object s", b'5\n'),
|
||||
("MIN", "select min(s.age) from S3Object s", b'0\n'),
|
||||
("SUM", "select sum(s.age) from S3Object s", b'12\n'),
|
||||
# Conditional functions
|
||||
("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
|
||||
("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
|
||||
# Conversion functions
|
||||
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s",
|
||||
b'3.0\n4.0\n5.0\n\n0.0\n'),
|
||||
|
||||
]
|
||||
|
||||
|
|
@ -210,36 +221,49 @@ def test_sql_functions_date(client, log_output):
|
|||
"""
|
||||
|
||||
tests = [
|
||||
# DATE_ADD
|
||||
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'),
|
||||
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'),
|
||||
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'),
|
||||
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'),
|
||||
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'),
|
||||
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'),
|
||||
# DATE_DIFF
|
||||
("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
|
||||
("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
|
||||
("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'),
|
||||
# EXTRACT
|
||||
("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'),
|
||||
("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'),
|
||||
("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'),
|
||||
("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'),
|
||||
("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
|
||||
("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
|
||||
# TO_STRING
|
||||
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'),
|
||||
("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
|
||||
("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
|
||||
("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
|
||||
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'),
|
||||
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
|
||||
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
|
||||
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
|
||||
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'),
|
||||
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'),
|
||||
("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)),
|
||||
# DATE_ADD
|
||||
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
|
||||
b'2022-01-02T03:04:05.006+07:30\n'),
|
||||
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
|
||||
b'2017-02-02T03:04:05.006+07:30\n'),
|
||||
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s",
|
||||
b'2017-01-01T03:04:05.006+07:30\n'),
|
||||
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
|
||||
b'2017-01-02T04:04:05.006+07:30\n'),
|
||||
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
|
||||
b'2017-01-02T03:09:05.006+07:30\n'),
|
||||
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
|
||||
b'2017-01-02T03:04:10.006+07:30\n'),
|
||||
# DATE_DIFF
|
||||
("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
|
||||
("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
|
||||
("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'),
|
||||
# EXTRACT
|
||||
("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'),
|
||||
("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'),
|
||||
("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'),
|
||||
("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'),
|
||||
("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
|
||||
("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
|
||||
# TO_STRING
|
||||
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s",
|
||||
b'"January 2, 2017"\n'),
|
||||
("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
|
||||
("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
|
||||
("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
|
||||
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s",
|
||||
b'"January 2, 2017 3:4 AM"\n'),
|
||||
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
|
||||
b'2017-01-02T3:4:05+0730\n'),
|
||||
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
|
||||
b'2017-01-02T3:4:05+0730\n'),
|
||||
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s",
|
||||
b'2017-01-02T3:4:05+0730\n'),
|
||||
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s",
|
||||
b'2017-01-02T3:4:05+07:30\n'),
|
||||
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s",
|
||||
b'2017-01-02T03:04:05.006+07:30\n'),
|
||||
("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)),
|
||||
|
||||
]
|
||||
|
||||
|
|
@ -253,6 +277,7 @@ def test_sql_functions_date(client, log_output):
|
|||
# Test passes
|
||||
print(log_output.json_report())
|
||||
|
||||
|
||||
def test_sql_functions_string(client, log_output):
|
||||
|
||||
json_testfile = """
|
||||
|
|
@ -262,23 +287,26 @@ def test_sql_functions_string(client, log_output):
|
|||
"""
|
||||
|
||||
tests = [
|
||||
# CHAR_LENGTH
|
||||
("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
|
||||
("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
|
||||
# LOWER
|
||||
("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
|
||||
# SUBSTRING
|
||||
("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'),
|
||||
("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'),
|
||||
("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
|
||||
# TRIM
|
||||
("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
|
||||
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'),
|
||||
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'),
|
||||
("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
|
||||
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
|
||||
# UPPER
|
||||
("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'),
|
||||
# CHAR_LENGTH
|
||||
("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
|
||||
("CHARACTER_LENGTH",
|
||||
"select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
|
||||
# LOWER
|
||||
("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
|
||||
# SUBSTRING
|
||||
("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'),
|
||||
("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'),
|
||||
("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
|
||||
# TRIM
|
||||
("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
|
||||
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2",
|
||||
b'\tfoobar\t \n'),
|
||||
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2",
|
||||
b' \tfoobar\t\n'),
|
||||
("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
|
||||
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
|
||||
# UPPER
|
||||
("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'),
|
||||
]
|
||||
|
||||
try:
|
||||
|
|
@ -297,14 +325,15 @@ def test_sql_datatypes(client, log_output):
|
|||
{"name": "John"}
|
||||
"""
|
||||
tests = [
|
||||
("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'),
|
||||
("int", "select CAST('13' AS INT) from S3Object", b'13\n'),
|
||||
("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'),
|
||||
("string", "select CAST(true AS STRING) from S3Object", b'true\n'),
|
||||
("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
|
||||
("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
|
||||
("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
|
||||
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'),
|
||||
("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'),
|
||||
("int", "select CAST('13' AS INT) from S3Object", b'13\n'),
|
||||
("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'),
|
||||
("string", "select CAST(true AS STRING) from S3Object", b'true\n'),
|
||||
("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
|
||||
("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
|
||||
("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
|
||||
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object",
|
||||
b'2007-04-05T14:30Z\n'),
|
||||
]
|
||||
|
||||
try:
|
||||
|
|
@ -323,14 +352,17 @@ def test_sql_select(client, log_output):
|
|||
json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" }
|
||||
{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }"""
|
||||
tests = [
|
||||
("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
|
||||
("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
|
||||
("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
|
||||
("select_4", "select s.line from S3Object as s", b'\n\n'),
|
||||
("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
|
||||
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
|
||||
("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'),
|
||||
("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'),
|
||||
("select_1", "select * from S3Object",
|
||||
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
|
||||
("select_2", "select * from S3Object s",
|
||||
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
|
||||
("select_3", "select * from S3Object as s",
|
||||
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
|
||||
("select_4", "select s.line from S3Object as s", b'\n\n'),
|
||||
("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
|
||||
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
|
||||
("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'),
|
||||
("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'),
|
||||
]
|
||||
|
||||
try:
|
||||
|
|
@ -343,23 +375,29 @@ def test_sql_select(client, log_output):
|
|||
# Test passes
|
||||
print(log_output.json_report())
|
||||
|
||||
|
||||
def test_sql_select_json(client, log_output):
|
||||
json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]}
|
||||
{ "created": "June 27", "modified": "July 6" }
|
||||
"""
|
||||
tests = [
|
||||
("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
|
||||
("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
|
||||
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
|
||||
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
|
||||
("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
|
||||
("select_1", "SELECT id FROM S3Object[*].Rules[*].id",
|
||||
b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
|
||||
("select_2",
|
||||
"SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
|
||||
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d",
|
||||
b'{}\n{"created":"June 27","modified":"July 6"}\n'),
|
||||
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]",
|
||||
b'{}\n{"created":"June 27","modified":"July 6"}\n'),
|
||||
("select_5",
|
||||
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
|
||||
]
|
||||
|
||||
input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT"))
|
||||
output_serialization = OutputSerialization(json=JsonOutput())
|
||||
try:
|
||||
test_sql_expressions_custom_input_output(client, json_testcontent,
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
except Exception as select_err:
|
||||
raise select_err
|
||||
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
|
||||
|
|
@ -374,20 +412,20 @@ def test_sql_select_csv_no_header(client, log_output):
|
|||
val4,val5,val6
|
||||
"""
|
||||
tests = [
|
||||
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
|
||||
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
|
||||
]
|
||||
|
||||
input_serialization=InputSerialization(
|
||||
csv=CSVInput(
|
||||
FileHeaderInfo="NONE",
|
||||
AllowQuotedRecordDelimiter="FALSE",
|
||||
),
|
||||
)
|
||||
input_serialization = InputSerialization(
|
||||
csv=CSVInput(
|
||||
FileHeaderInfo="NONE",
|
||||
AllowQuotedRecordDelimiter="FALSE",
|
||||
),
|
||||
)
|
||||
|
||||
output_serialization=OutputSerialization(csv=CSVOutput())
|
||||
output_serialization = OutputSerialization(csv=CSVOutput())
|
||||
try:
|
||||
test_sql_expressions_custom_input_output(client, json_testcontent,
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
except Exception as select_err:
|
||||
raise select_err
|
||||
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
|
||||
|
|
@ -395,5 +433,3 @@ val4,val5,val6
|
|||
|
||||
# Test passes
|
||||
print(log_output.json_report())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,12 +16,19 @@
|
|||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from sys import exit
|
||||
import sys
|
||||
from csv import (test_csv_input_custom_quote_char,
|
||||
test_csv_output_custom_quote_char)
|
||||
|
||||
from minio import Minio
|
||||
|
||||
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
|
||||
test_sql_functions_date, test_sql_functions_string,
|
||||
test_sql_operators, test_sql_operators_precedence,
|
||||
test_sql_select, test_sql_select_csv_no_header,
|
||||
test_sql_select_json)
|
||||
from utils import LogOutput
|
||||
from sql_ops import *
|
||||
from csv import *
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
|
|
@ -39,48 +46,56 @@ def main():
|
|||
secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
|
||||
secure = True
|
||||
|
||||
client = Minio(server_endpoint, access_key, secret_key, secure=False)
|
||||
client = Minio(server_endpoint, access_key, secret_key, secure=secure)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char')
|
||||
log_output = LogOutput(client.select_object_content,
|
||||
'test_csv_input_quote_char')
|
||||
test_csv_input_custom_quote_char(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char')
|
||||
log_output = LogOutput(client.select_object_content,
|
||||
'test_csv_output_quote_char')
|
||||
test_csv_output_custom_quote_char(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_operators')
|
||||
log_output = LogOutput(
|
||||
client.select_object_content, 'test_sql_operators')
|
||||
test_sql_operators(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence')
|
||||
log_output = LogOutput(client.select_object_content,
|
||||
'test_sql_operators_precedence')
|
||||
test_sql_operators_precedence(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv')
|
||||
log_output = LogOutput(client.select_object_content,
|
||||
'test_sql_functions_agg_cond_conv')
|
||||
test_sql_functions_agg_cond_conv(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_functions_date')
|
||||
log_output = LogOutput(
|
||||
client.select_object_content, 'test_sql_functions_date')
|
||||
test_sql_functions_date(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_functions_string')
|
||||
log_output = LogOutput(client.select_object_content,
|
||||
'test_sql_functions_string')
|
||||
test_sql_functions_string(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_datatypes')
|
||||
log_output = LogOutput(
|
||||
client.select_object_content, 'test_sql_datatypes')
|
||||
test_sql_datatypes(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_select')
|
||||
test_sql_select(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_select_json')
|
||||
log_output = LogOutput(
|
||||
client.select_object_content, 'test_sql_select_json')
|
||||
test_sql_select_json(client, log_output)
|
||||
|
||||
log_output = LogOutput(client.select_object_content, 'test_sql_select_csv')
|
||||
log_output = LogOutput(
|
||||
client.select_object_content, 'test_sql_select_csv')
|
||||
test_sql_select_csv_no_header(client, log_output)
|
||||
|
||||
except Exception as err:
|
||||
print(log_output.json_report(err))
|
||||
exit(1)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Execute only if run as a script
|
||||
main()
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,11 +15,12 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import uuid
|
||||
import inspect
|
||||
import json
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
|
||||
|
||||
class LogOutput(object):
|
||||
"""
|
||||
|
|
@ -100,7 +101,6 @@ class LogOutput(object):
|
|||
def generate_bucket_name():
|
||||
return "s3select-test-" + str(uuid.uuid4())
|
||||
|
||||
|
||||
def generate_object_name():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ type CallStats struct {
|
|||
// RequestInfo represents trace of http request
|
||||
type RequestInfo struct {
|
||||
Time time.Time `json:"time"`
|
||||
Proto string `json:"proto"`
|
||||
Method string `json:"method"`
|
||||
Path string `json:"path,omitempty"`
|
||||
RawQuery string `json:"rawquery,omitempty"`
|
||||
|
|
|
|||
Loading…
Reference in a new issue