Add startup testing for Wasm Pinecone build (#1910)

* Only include go-sqlite3 on the relevant binaries

* The driver name is always sqlite3 now

* Update to matrix-org/go-sqlite3-js@e537baa

* Add initial Wasm test harness

* Upgrade go-sqlite3-js

This fixes an error about semicolons in single statements.

* Add browser-like WebSocket API for testing

* Upgrade go-sqlite3-js

This upgrade includes printing panic messages next to stacks.

* Run for all PRs targeting any branch

* Use manual Node caching

* Temporarily run for all pushes

* Use npm ci instead of install

* Use HTTPS auth for repo packages

* Match path style from build.sh

* update utp

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
This commit is contained in:
J. Ryan Stinnett 2021-07-20 12:14:58 +01:00 committed by GitHub
parent f63068df3b
commit 034f5ae3f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 234 additions and 13 deletions

49
.github/workflows/wasm.yml vendored Normal file
View File

@ -0,0 +1,49 @@
name: WebAssembly
on:
push:
pull_request:
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.16.5
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install Node
uses: actions/setup-node@v2
with:
node-version: 14
- uses: actions/cache@v2
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Reconfigure Git to use HTTPS auth for repo packages
run: >
git config --global url."https://github.com/".insteadOf
ssh://git@github.com/
- name: Install test dependencies
working-directory: ./test/wasm
run: npm ci
- name: Test
run: ./test-dendritejs.sh

7
.gitignore vendored
View File

@ -3,6 +3,9 @@
# Hidden files # Hidden files
.* .*
# Allow GitHub config
!.github
# Downloads # Downloads
/.downloads /.downloads
@ -36,6 +39,7 @@ _testmain.go
*.exe *.exe
*.test *.test
*.prof *.prof
*.wasm
# Generated keys # Generated keys
*.pem *.pem
@ -53,3 +57,6 @@ dendrite.yaml
# Generated code # Generated code
cmd/dendrite-demo-yggdrasil/embed/fs*.go cmd/dendrite-demo-yggdrasil/embed/fs*.go
# Test dependencies
test/wasm/node_modules

View File

@ -1,4 +1,4 @@
#!/bin/sh -eu #!/bin/sh -eu
export GIT_COMMIT=$(git rev-list -1 HEAD) && \ export GIT_COMMIT=$(git rev-list -1 HEAD) && \
GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o bin/main.wasm ./cmd/dendritejs-pinecone

View File

@ -21,4 +21,4 @@ mkdir -p bin
CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

View File

@ -144,6 +144,13 @@ func generateKey() ed25519.PrivateKey {
} }
func main() { func main() {
startup()
// We want to block forever to let the fetch and libp2p handler serve the APIs
select {}
}
func startup() {
sk := generateKey() sk := generateKey()
pk := sk.Public().(ed25519.PublicKey) pk := sk.Public().(ed25519.PublicKey)
@ -251,7 +258,4 @@ func main() {
} }
} }
}() }()
// We want to block forever to let the fetch and libp2p handler serve the APIs
select {}
} }

View File

@ -0,0 +1,25 @@
// Copyright 2021 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build wasm
package main
import (
"testing"
)
func TestStartup(t *testing.T) {
startup()
}

View File

@ -2,14 +2,23 @@
These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up. These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
### Dendrite ### Dendrite
#### Build
- The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`. - The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
- Build it and copy assets to riot-web. - Build it and copy assets to riot-web.
``` ```
$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs $ ./build-dendritejs.sh
$ cp main.wasm ../riot-web/src/vector/dendrite.wasm $ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
```
#### Test
To check that the Dendrite side is working well as Wasm, you can run the
Wasm-specific tests:
```
$ ./test-dendritejs.sh
``` ```
### Rendezvous ### Rendezvous

2
go.mod
View File

@ -37,7 +37,7 @@ require (
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb
github.com/morikuni/aec v1.0.0 // indirect github.com/morikuni/aec v1.0.0 // indirect
github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
github.com/opentracing/opentracing-go v1.2.0 github.com/opentracing/opentracing-go v1.2.0

6
go.sum
View File

@ -118,9 +118,7 @@ github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgw
github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo=
github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw= github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw=
github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y=
github.com/anacrolix/missinggo/perf v1.0.0 h1:7ZOGYziGEBytW49+KmYGTaNfnwUqP1HBsy6BqESAJVw=
github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
github.com/anacrolix/sync v0.2.0 h1:oRe22/ZB+v7v/5Mbc4d2zE0AXEZy0trKyKLjqYOt6tY=
github.com/anacrolix/sync v0.2.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g= github.com/anacrolix/sync v0.2.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g=
github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk= github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk=
@ -1186,8 +1184,8 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b h1:XNm+Ks3bVziRJxcMaIbzumWEw7l52z9Rek6cMHgln1g= github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2 h1:txJOiDxsypF8RbzbcyOD3ovip+uUWNZE/Zo7qLdARZQ=
github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b/go.mod h1:ylsx0342RjGHjOoVKhR/wz/7Lhiusonihfj4QLxEMcU= github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww= github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=

3
test-dendritejs.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh -eu
GOOS=js GOARCH=wasm go test -v -exec "$(pwd)/test/wasm/index.js" ./cmd/dendritejs-pinecone

52
test/wasm/index.js Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env node
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
const fs = require('fs');
const path = require('path');
const childProcess = require('child_process');
(async function() {
// sql.js
const initSqlJs = require('sql.js');
await initSqlJs().then(SQL => {
global._go_sqlite = SQL;
console.log("Loaded sqlite")
});
// dendritejs expects to write to `/idb` so we create that here
// Since this is testing only, we use the default in-memory FS
global._go_sqlite.FS.mkdir("/idb");
// WebSocket
const WebSocket = require('isomorphic-ws');
global.WebSocket = WebSocket;
// Load the generic Go Wasm exec helper inline to trigger built-in run call
// This approach avoids copying `wasm_exec.js` into the repo, which is nice
// to aim for since it can differ between Go versions.
const goRoot = await new Promise((resolve, reject) => {
childProcess.execFile('go', ['env', 'GOROOT'], (err, out) => {
if (err) {
reject("Can't find go");
}
resolve(out.trim());
});
});
const execPath = path.join(goRoot, 'misc/wasm/wasm_exec.js');
const execCode = fs.readFileSync(execPath, 'utf8');
eval(execCode);
})();

67
test/wasm/package-lock.json generated Normal file
View File

@ -0,0 +1,67 @@
{
"name": "wasm",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"dependencies": {
"isomorphic-ws": "^4.0.1",
"sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
"ws": "^7.5.2"
}
},
"node_modules/isomorphic-ws": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
"integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
"peerDependencies": {
"ws": "*"
}
},
"node_modules/sql.js": {
"version": "1.5.0",
"resolved": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
"integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
"license": "MIT"
},
"node_modules/ws": {
"version": "7.5.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
"integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
"engines": {
"node": ">=8.3.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": "^5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
}
},
"dependencies": {
"isomorphic-ws": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
"integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
"requires": {}
},
"sql.js": {
"version": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
"integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
"from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae"
},
"ws": {
"version": "7.5.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
"integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
"requires": {}
}
}
}

7
test/wasm/package.json Normal file
View File

@ -0,0 +1,7 @@
{
"dependencies": {
"isomorphic-ws": "^4.0.1",
"sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
"ws": "^7.5.2"
}
}