Browse Source
tools/sdk-live-tests/run.sh builds the server with unsafe_test_mode and runs the live tests of the SDK's e2e/wasm suite. For SDK versions whose tests log in to one given account, each user test vector with a master password is seeded with its vault on a fresh server; for versions that seed their own accounts, the tests get the seeder's address. Every vector runs, then a summary. The workflow runs it by hand, and on changes to the seeder or the runner.pull/7953/head
committed by
GitHub
3 changed files with 274 additions and 0 deletions
@ -0,0 +1,74 @@ |
|||
name: SDK live tests |
|||
permissions: {} |
|||
|
|||
# Runs the live-server end-to-end tests of bitwarden/sdk-internal against this branch, see tools/sdk-live-tests/run.sh. |
|||
# It builds the SDK too, so it only runs by hand and on changes to the test seeder or the runner. |
|||
on: |
|||
workflow_dispatch: |
|||
inputs: |
|||
sdk_ref: |
|||
description: "Branch, tag or commit of bitwarden/sdk-internal to test with" |
|||
required: true |
|||
default: "main" |
|||
pull_request: |
|||
paths: |
|||
- ".github/workflows/sdk-live-tests.yml" |
|||
- "src/api/test_seeder.rs" |
|||
- "tools/sdk-live-tests/**" |
|||
|
|||
defaults: |
|||
run: |
|||
shell: bash |
|||
|
|||
jobs: |
|||
sdk-live-tests: |
|||
name: SDK live tests |
|||
runs-on: ubuntu-24.04 |
|||
timeout-minutes: 90 |
|||
steps: |
|||
- name: "Install dependencies Ubuntu" |
|||
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential libssl-dev pkg-config |
|||
|
|||
- name: "Checkout" |
|||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
|||
with: |
|||
persist-credentials: false |
|||
|
|||
- name: "Checkout sdk-internal" |
|||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
|||
with: |
|||
repository: bitwarden/sdk-internal |
|||
ref: ${{ inputs.sdk_ref || 'main' }} |
|||
path: sdk-internal |
|||
persist-credentials: false |
|||
|
|||
# Each checkout pins its own toolchain in rust-toolchain.toml |
|||
- name: "Install toolchains" |
|||
run: | |
|||
rustup toolchain install |
|||
cd sdk-internal |
|||
rustup toolchain install |
|||
rustup target add wasm32-unknown-unknown |
|||
rustup component add rust-src |
|||
|
|||
- name: "Cache Rust builds" |
|||
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 |
|||
with: |
|||
workspaces: | |
|||
. -> target |
|||
sdk-internal -> target |
|||
|
|||
- name: "Setup Node" |
|||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 |
|||
with: |
|||
node-version-file: sdk-internal/.nvmrc |
|||
|
|||
- name: "Run the SDK live tests" |
|||
run: tools/sdk-live-tests/run.sh --sdk-dir sdk-internal |
|||
|
|||
- name: "Upload logs" |
|||
if: ${{ !cancelled() }} |
|||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 |
|||
with: |
|||
name: sdk-live-tests-logs |
|||
path: target/sdk-live-tests/**/*.log |
|||
@ -0,0 +1,141 @@ |
|||
#!/usr/bin/env bash |
|||
# Runs the live-server end-to-end tests of bitwarden/sdk-internal (e2e/wasm) against vaultwarden. |
|||
# |
|||
# Usage: tools/sdk-live-tests/run.sh --sdk-dir <path> [--skip-sdk-build] [--mode vectors|seeder] [-- <vitest args>] |
|||
# |
|||
# The server is built with `unsafe_test_mode`, whose test seeder (src/api/test_seeder.rs) creates the test accounts. |
|||
# - vectors: for SDK versions whose live tests log in to one given account. Every user test vector with a master |
|||
# password gets a fresh server, is seeded there with its vault, and the tests run on it (they rotate its keys). |
|||
# - seeder: for SDK versions whose tests seed their own accounts through BW_LIVE_SEEDER_URL. One server, one run. |
|||
# The mode follows what the SDK's e2e/wasm/live/config.ts reads, unless --mode says otherwise. Arguments after `--` go |
|||
# to vitest, e.g. `-- tests/user-crypto-management/rotate-user-keys.live.test.ts` to run one file. |
|||
# |
|||
# Every vector runs even after a failure; a summary follows, and the exit code is 1 if any failed. The server logs are |
|||
# in <target dir>/sdk-live-tests/. BW_LIVE_CLIENT_VERSION defaults to 2026.10.1, the minimum for V2 accounts. |
|||
# |
|||
# Needs: the SDK's Rust toolchain with the wasm32-unknown-unknown target and rust-src, Node.js >= 22.12, curl, python3. |
|||
set -euo pipefail |
|||
|
|||
VW_DIR="$(cd "$(dirname "$0")/../.." && pwd)" |
|||
TARGET_DIR="${CARGO_TARGET_DIR:-${VW_DIR}/target}" |
|||
OUT_DIR="${TARGET_DIR}/sdk-live-tests" |
|||
PORT=8099 |
|||
URL="http://127.0.0.1:${PORT}" |
|||
SEEDER_URL="${URL}/test/seeder" |
|||
SEEDER_USERNAME="sdk-live-tests" |
|||
SEEDER_PASSWORD="$(python3 -c 'import secrets; print(secrets.token_urlsafe(24))')" |
|||
SKIP=3 # seed_vector.py's exit code for a vector the tests can't use |
|||
export BW_LIVE_CLIENT_VERSION="${BW_LIVE_CLIENT_VERSION:-2026.10.1}" |
|||
|
|||
SDK_DIR="" |
|||
BUILD_SDK=1 |
|||
MODE="" |
|||
VITEST_ARGS=() |
|||
while [[ $# -gt 0 ]]; do |
|||
case "$1" in |
|||
--sdk-dir) SDK_DIR="$(cd "$2" && pwd)"; shift 2 ;; |
|||
--skip-sdk-build) BUILD_SDK=0; shift ;; |
|||
--mode) MODE="$2"; shift 2 ;; |
|||
--) shift; VITEST_ARGS=("$@"); break ;; |
|||
*) echo "Unknown argument: $1" >&2; exit 2 ;; |
|||
esac |
|||
done |
|||
[[ -n "${SDK_DIR}" ]] || { echo "--sdk-dir is required" >&2; exit 2; } |
|||
TESTS_DIR="${SDK_DIR}/e2e/wasm" |
|||
[[ -f "${TESTS_DIR}/live/config.ts" ]] || { echo "${TESTS_DIR}/live/config.ts not found: no live tests in this SDK" >&2; exit 2; } |
|||
if [[ -z "${MODE}" ]]; then |
|||
if grep -q BW_LIVE_SEEDER_URL "${TESTS_DIR}/live/config.ts"; then MODE=seeder; else MODE=vectors; fi |
|||
fi |
|||
[[ "${MODE}" == vectors || "${MODE}" == seeder ]] || { echo "--mode must be vectors or seeder" >&2; exit 2; } |
|||
echo "Mode: ${MODE}, client version ${BW_LIVE_CLIENT_VERSION}" |
|||
|
|||
(cd "${VW_DIR}" && cargo build --features sqlite,unsafe_test_mode) |
|||
if [[ ${BUILD_SDK} -eq 1 ]]; then |
|||
bash "${SDK_DIR}/crates/bitwarden-wasm-internal/build.sh" |
|||
(cd "${TESTS_DIR}" && npm ci) |
|||
fi |
|||
|
|||
server="" |
|||
trap 'if [[ -n "${server}" ]]; then kill "${server}" 2>/dev/null || true; fi' EXIT |
|||
|
|||
# Starts a server on an empty database in $OUT_DIR/$1 |
|||
start_server() { |
|||
local data_dir="${OUT_DIR}/$1" |
|||
rm -rf "${data_dir}" && mkdir -p "${data_dir}" |
|||
if curl -s -o /dev/null "${URL}/alive"; then |
|||
echo "Something already answers on port ${PORT}" >&2 |
|||
return 1 |
|||
fi |
|||
ENV_FILE=/dev/null DATA_FOLDER="${data_dir}" DATABASE_URL="sqlite://${data_dir}/db.sqlite3" \ |
|||
ROCKET_ADDRESS=127.0.0.1 ROCKET_PORT="${PORT}" DOMAIN="${URL}" WEB_VAULT_ENABLED=false \ |
|||
SIGNUPS_ALLOWED=true SIGNUPS_VERIFY=false LOGIN_RATELIMIT_MAX_BURST=1000 \ |
|||
TEST_SEEDER_USERNAME="${SEEDER_USERNAME}" TEST_SEEDER_PASSWORD="${SEEDER_PASSWORD}" \ |
|||
"${TARGET_DIR}/debug/vaultwarden" > "${data_dir}/vaultwarden.log" 2>&1 & |
|||
server=$! |
|||
local i |
|||
for i in $(seq 1 60); do |
|||
if ! kill -0 "${server}" 2>/dev/null; then break; fi |
|||
if curl -sf -o /dev/null "${URL}/alive"; then return 0; fi |
|||
sleep 1 |
|||
done |
|||
echo "The server didn't start; the end of ${data_dir}/vaultwarden.log:" >&2 |
|||
tail -50 "${data_dir}/vaultwarden.log" >&2 |
|||
stop_server |
|||
return 1 |
|||
} |
|||
|
|||
stop_server() { |
|||
if [[ -n "${server}" ]]; then |
|||
kill "${server}" 2>/dev/null || true |
|||
wait "${server}" 2>/dev/null || true |
|||
server="" |
|||
fi |
|||
} |
|||
|
|||
run_tests() { |
|||
(cd "${TESTS_DIR}" && BW_LIVE_SERVER_URL="${URL}" "$@" npm run test:live -- "${VITEST_ARGS[@]}") |
|||
} |
|||
|
|||
results=() |
|||
failed=0 |
|||
if [[ "${MODE}" == seeder ]]; then |
|||
if start_server seeder && run_tests env BW_LIVE_SEEDER_URL="${SEEDER_URL}" \ |
|||
BW_LIVE_SEEDER_USERNAME="${SEEDER_USERNAME}" BW_LIVE_SEEDER_PASSWORD="${SEEDER_PASSWORD}"; then |
|||
results+=("seeder: pass") |
|||
else |
|||
results+=("seeder: FAIL") |
|||
failed=1 |
|||
fi |
|||
stop_server |
|||
else |
|||
for vector_file in "${SDK_DIR}"/test-vectors/users/*.json; do |
|||
vector="$(basename "${vector_file}" .json)" |
|||
echo "=== ${vector}" |
|||
if ! start_server "${vector}"; then |
|||
results+=("${vector}: FAIL (server didn't start)") |
|||
failed=1 |
|||
continue |
|||
fi |
|||
status=0 |
|||
credentials="$(python3 -I "${VW_DIR}/tools/sdk-live-tests/seed_vector.py" "${vector_file}" "${SEEDER_URL}" \ |
|||
"${SEEDER_USERNAME}" "${SEEDER_PASSWORD}")" || status=$? |
|||
if [[ ${status} -eq ${SKIP} ]]; then |
|||
results+=("${vector}: skip (${credentials})") |
|||
elif [[ ${status} -ne 0 ]]; then |
|||
results+=("${vector}: FAIL (seeding)") |
|||
failed=1 |
|||
elif run_tests env BW_LIVE_EMAIL="$(sed -n 1p <<< "${credentials}")" \ |
|||
BW_LIVE_PASSWORD="$(sed -n 2p <<< "${credentials}")"; then |
|||
results+=("${vector}: pass") |
|||
else |
|||
results+=("${vector}: FAIL") |
|||
failed=1 |
|||
fi |
|||
stop_server |
|||
done |
|||
fi |
|||
|
|||
echo |
|||
echo "=== Summary (logs in ${OUT_DIR})" |
|||
printf '%s\n' "${results[@]}" |
|||
exit "${failed}" |
|||
@ -0,0 +1,59 @@ |
|||
"""Seeds one of the SDK's user test vectors, vault included, through vaultwarden's test seeder. |
|||
|
|||
Usage: seed_vector.py <vector.json> <seeder url> <seeder username> <seeder password> |
|||
|
|||
Prints the account's email and password, one per line. Exits 3, printing why, for a vector the tests can't log in |
|||
with (no master password), and 1 if the seeder refuses it. |
|||
""" |
|||
|
|||
import base64 |
|||
import json |
|||
import sys |
|||
import urllib.error |
|||
import urllib.request |
|||
import uuid |
|||
|
|||
SKIP = 3 |
|||
|
|||
|
|||
def main() -> int: |
|||
vector_file, seeder_url, username, password = sys.argv[1:5] |
|||
with open(vector_file, encoding="utf-8") as f: |
|||
vector = json.load(f) |
|||
|
|||
has_master_password = any("masterPasswordUnlock" in method for method in vector.get("unlockMethods", [])) |
|||
if not has_master_password or "password" not in vector.get("account", {}): |
|||
print("no master password") |
|||
return SKIP |
|||
|
|||
# Without attachments: the rotation test compares the vault before and after, and our sync gives each attachment |
|||
# a link with a fresh download token every time (upstream's is the same on every sync). A vector's legacy |
|||
# attachments, without a key of their own, would also make the SDK refuse to rotate at all. |
|||
for cipher in vector.get("vault", {}).get("ciphers", []): |
|||
cipher.get("encrypted", {}).pop("attachments", None) |
|||
|
|||
credentials = base64.b64encode(f"{username}:{password}".encode()).decode() |
|||
request = urllib.request.Request( |
|||
f"{seeder_url}/seed", |
|||
data=json.dumps({"template": "TestVectorUserScene", "arguments": vector}).encode(), |
|||
method="POST", |
|||
headers={ |
|||
"Authorization": f"Basic {credentials}", |
|||
"Content-Type": "application/json", |
|||
"X-Play-Id": f"sdk-live-tests-{uuid.uuid4()}", |
|||
}, |
|||
) |
|||
try: |
|||
with urllib.request.urlopen(request) as response: |
|||
result = json.load(response)["result"] |
|||
except urllib.error.HTTPError as e: |
|||
print(f"The seeder refused {vector_file}: {e.code} {e.read().decode(errors='replace')}", file=sys.stderr) |
|||
return 1 |
|||
|
|||
print(result["email"]) |
|||
print(vector["account"]["password"]) |
|||
return 0 |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
sys.exit(main()) |
|||
Loading…
Reference in new issue