Browse Source
tools/sdk-live-tests/run.sh builds the server with unsafe_test_mode and runs the live tests of the SDK's e2e/wasm suite. For SDK versions whose tests log in to one given account, each user test vector with a master password is seeded with its vault on a fresh server; for versions that seed their own accounts, the tests get the seeder's address. Every vector runs, then a summary. The workflow runs it by hand, and on changes to the seeder or the runner.pull/7953/head
committed by
GitHub
3 changed files with 274 additions and 0 deletions
@ -0,0 +1,74 @@ |
|||||
|
name: SDK live tests |
||||
|
permissions: {} |
||||
|
|
||||
|
# Runs the live-server end-to-end tests of bitwarden/sdk-internal against this branch, see tools/sdk-live-tests/run.sh. |
||||
|
# It builds the SDK too, so it only runs by hand and on changes to the test seeder or the runner. |
||||
|
on: |
||||
|
workflow_dispatch: |
||||
|
inputs: |
||||
|
sdk_ref: |
||||
|
description: "Branch, tag or commit of bitwarden/sdk-internal to test with" |
||||
|
required: true |
||||
|
default: "main" |
||||
|
pull_request: |
||||
|
paths: |
||||
|
- ".github/workflows/sdk-live-tests.yml" |
||||
|
- "src/api/test_seeder.rs" |
||||
|
- "tools/sdk-live-tests/**" |
||||
|
|
||||
|
defaults: |
||||
|
run: |
||||
|
shell: bash |
||||
|
|
||||
|
jobs: |
||||
|
sdk-live-tests: |
||||
|
name: SDK live tests |
||||
|
runs-on: ubuntu-24.04 |
||||
|
timeout-minutes: 90 |
||||
|
steps: |
||||
|
- name: "Install dependencies Ubuntu" |
||||
|
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential libssl-dev pkg-config |
||||
|
|
||||
|
- name: "Checkout" |
||||
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
||||
|
with: |
||||
|
persist-credentials: false |
||||
|
|
||||
|
- name: "Checkout sdk-internal" |
||||
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
||||
|
with: |
||||
|
repository: bitwarden/sdk-internal |
||||
|
ref: ${{ inputs.sdk_ref || 'main' }} |
||||
|
path: sdk-internal |
||||
|
persist-credentials: false |
||||
|
|
||||
|
# Each checkout pins its own toolchain in rust-toolchain.toml |
||||
|
- name: "Install toolchains" |
||||
|
run: | |
||||
|
rustup toolchain install |
||||
|
cd sdk-internal |
||||
|
rustup toolchain install |
||||
|
rustup target add wasm32-unknown-unknown |
||||
|
rustup component add rust-src |
||||
|
|
||||
|
- name: "Cache Rust builds" |
||||
|
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 |
||||
|
with: |
||||
|
workspaces: | |
||||
|
. -> target |
||||
|
sdk-internal -> target |
||||
|
|
||||
|
- name: "Setup Node" |
||||
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 |
||||
|
with: |
||||
|
node-version-file: sdk-internal/.nvmrc |
||||
|
|
||||
|
- name: "Run the SDK live tests" |
||||
|
run: tools/sdk-live-tests/run.sh --sdk-dir sdk-internal |
||||
|
|
||||
|
- name: "Upload logs" |
||||
|
if: ${{ !cancelled() }} |
||||
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 |
||||
|
with: |
||||
|
name: sdk-live-tests-logs |
||||
|
path: target/sdk-live-tests/**/*.log |
||||
@ -0,0 +1,141 @@ |
|||||
|
#!/usr/bin/env bash |
||||
|
# Runs the live-server end-to-end tests of bitwarden/sdk-internal (e2e/wasm) against vaultwarden. |
||||
|
# |
||||
|
# Usage: tools/sdk-live-tests/run.sh --sdk-dir <path> [--skip-sdk-build] [--mode vectors|seeder] [-- <vitest args>] |
||||
|
# |
||||
|
# The server is built with `unsafe_test_mode`, whose test seeder (src/api/test_seeder.rs) creates the test accounts. |
||||
|
# - vectors: for SDK versions whose live tests log in to one given account. Every user test vector with a master |
||||
|
# password gets a fresh server, is seeded there with its vault, and the tests run on it (they rotate its keys). |
||||
|
# - seeder: for SDK versions whose tests seed their own accounts through BW_LIVE_SEEDER_URL. One server, one run. |
||||
|
# The mode follows what the SDK's e2e/wasm/live/config.ts reads, unless --mode says otherwise. Arguments after `--` go |
||||
|
# to vitest, e.g. `-- tests/user-crypto-management/rotate-user-keys.live.test.ts` to run one file. |
||||
|
# |
||||
|
# Every vector runs even after a failure; a summary follows, and the exit code is 1 if any failed. The server logs are |
||||
|
# in <target dir>/sdk-live-tests/. BW_LIVE_CLIENT_VERSION defaults to 2026.10.1, the minimum for V2 accounts. |
||||
|
# |
||||
|
# Needs: the SDK's Rust toolchain with the wasm32-unknown-unknown target and rust-src, Node.js >= 22.12, curl, python3. |
||||
|
set -euo pipefail |
||||
|
|
||||
|
VW_DIR="$(cd "$(dirname "$0")/../.." && pwd)" |
||||
|
TARGET_DIR="${CARGO_TARGET_DIR:-${VW_DIR}/target}" |
||||
|
OUT_DIR="${TARGET_DIR}/sdk-live-tests" |
||||
|
PORT=8099 |
||||
|
URL="http://127.0.0.1:${PORT}" |
||||
|
SEEDER_URL="${URL}/test/seeder" |
||||
|
SEEDER_USERNAME="sdk-live-tests" |
||||
|
SEEDER_PASSWORD="$(python3 -c 'import secrets; print(secrets.token_urlsafe(24))')" |
||||
|
SKIP=3 # seed_vector.py's exit code for a vector the tests can't use |
||||
|
export BW_LIVE_CLIENT_VERSION="${BW_LIVE_CLIENT_VERSION:-2026.10.1}" |
||||
|
|
||||
|
SDK_DIR="" |
||||
|
BUILD_SDK=1 |
||||
|
MODE="" |
||||
|
VITEST_ARGS=() |
||||
|
while [[ $# -gt 0 ]]; do |
||||
|
case "$1" in |
||||
|
--sdk-dir) SDK_DIR="$(cd "$2" && pwd)"; shift 2 ;; |
||||
|
--skip-sdk-build) BUILD_SDK=0; shift ;; |
||||
|
--mode) MODE="$2"; shift 2 ;; |
||||
|
--) shift; VITEST_ARGS=("$@"); break ;; |
||||
|
*) echo "Unknown argument: $1" >&2; exit 2 ;; |
||||
|
esac |
||||
|
done |
||||
|
[[ -n "${SDK_DIR}" ]] || { echo "--sdk-dir is required" >&2; exit 2; } |
||||
|
TESTS_DIR="${SDK_DIR}/e2e/wasm" |
||||
|
[[ -f "${TESTS_DIR}/live/config.ts" ]] || { echo "${TESTS_DIR}/live/config.ts not found: no live tests in this SDK" >&2; exit 2; } |
||||
|
if [[ -z "${MODE}" ]]; then |
||||
|
if grep -q BW_LIVE_SEEDER_URL "${TESTS_DIR}/live/config.ts"; then MODE=seeder; else MODE=vectors; fi |
||||
|
fi |
||||
|
[[ "${MODE}" == vectors || "${MODE}" == seeder ]] || { echo "--mode must be vectors or seeder" >&2; exit 2; } |
||||
|
echo "Mode: ${MODE}, client version ${BW_LIVE_CLIENT_VERSION}" |
||||
|
|
||||
|
(cd "${VW_DIR}" && cargo build --features sqlite,unsafe_test_mode) |
||||
|
if [[ ${BUILD_SDK} -eq 1 ]]; then |
||||
|
bash "${SDK_DIR}/crates/bitwarden-wasm-internal/build.sh" |
||||
|
(cd "${TESTS_DIR}" && npm ci) |
||||
|
fi |
||||
|
|
||||
|
server="" |
||||
|
trap 'if [[ -n "${server}" ]]; then kill "${server}" 2>/dev/null || true; fi' EXIT |
||||
|
|
||||
|
# Starts a server on an empty database in $OUT_DIR/$1 |
||||
|
start_server() { |
||||
|
local data_dir="${OUT_DIR}/$1" |
||||
|
rm -rf "${data_dir}" && mkdir -p "${data_dir}" |
||||
|
if curl -s -o /dev/null "${URL}/alive"; then |
||||
|
echo "Something already answers on port ${PORT}" >&2 |
||||
|
return 1 |
||||
|
fi |
||||
|
ENV_FILE=/dev/null DATA_FOLDER="${data_dir}" DATABASE_URL="sqlite://${data_dir}/db.sqlite3" \ |
||||
|
ROCKET_ADDRESS=127.0.0.1 ROCKET_PORT="${PORT}" DOMAIN="${URL}" WEB_VAULT_ENABLED=false \ |
||||
|
SIGNUPS_ALLOWED=true SIGNUPS_VERIFY=false LOGIN_RATELIMIT_MAX_BURST=1000 \ |
||||
|
TEST_SEEDER_USERNAME="${SEEDER_USERNAME}" TEST_SEEDER_PASSWORD="${SEEDER_PASSWORD}" \ |
||||
|
"${TARGET_DIR}/debug/vaultwarden" > "${data_dir}/vaultwarden.log" 2>&1 & |
||||
|
server=$! |
||||
|
local i |
||||
|
for i in $(seq 1 60); do |
||||
|
if ! kill -0 "${server}" 2>/dev/null; then break; fi |
||||
|
if curl -sf -o /dev/null "${URL}/alive"; then return 0; fi |
||||
|
sleep 1 |
||||
|
done |
||||
|
echo "The server didn't start; the end of ${data_dir}/vaultwarden.log:" >&2 |
||||
|
tail -50 "${data_dir}/vaultwarden.log" >&2 |
||||
|
stop_server |
||||
|
return 1 |
||||
|
} |
||||
|
|
||||
|
stop_server() { |
||||
|
if [[ -n "${server}" ]]; then |
||||
|
kill "${server}" 2>/dev/null || true |
||||
|
wait "${server}" 2>/dev/null || true |
||||
|
server="" |
||||
|
fi |
||||
|
} |
||||
|
|
||||
|
run_tests() { |
||||
|
(cd "${TESTS_DIR}" && BW_LIVE_SERVER_URL="${URL}" "$@" npm run test:live -- "${VITEST_ARGS[@]}") |
||||
|
} |
||||
|
|
||||
|
results=() |
||||
|
failed=0 |
||||
|
if [[ "${MODE}" == seeder ]]; then |
||||
|
if start_server seeder && run_tests env BW_LIVE_SEEDER_URL="${SEEDER_URL}" \ |
||||
|
BW_LIVE_SEEDER_USERNAME="${SEEDER_USERNAME}" BW_LIVE_SEEDER_PASSWORD="${SEEDER_PASSWORD}"; then |
||||
|
results+=("seeder: pass") |
||||
|
else |
||||
|
results+=("seeder: FAIL") |
||||
|
failed=1 |
||||
|
fi |
||||
|
stop_server |
||||
|
else |
||||
|
for vector_file in "${SDK_DIR}"/test-vectors/users/*.json; do |
||||
|
vector="$(basename "${vector_file}" .json)" |
||||
|
echo "=== ${vector}" |
||||
|
if ! start_server "${vector}"; then |
||||
|
results+=("${vector}: FAIL (server didn't start)") |
||||
|
failed=1 |
||||
|
continue |
||||
|
fi |
||||
|
status=0 |
||||
|
credentials="$(python3 -I "${VW_DIR}/tools/sdk-live-tests/seed_vector.py" "${vector_file}" "${SEEDER_URL}" \ |
||||
|
"${SEEDER_USERNAME}" "${SEEDER_PASSWORD}")" || status=$? |
||||
|
if [[ ${status} -eq ${SKIP} ]]; then |
||||
|
results+=("${vector}: skip (${credentials})") |
||||
|
elif [[ ${status} -ne 0 ]]; then |
||||
|
results+=("${vector}: FAIL (seeding)") |
||||
|
failed=1 |
||||
|
elif run_tests env BW_LIVE_EMAIL="$(sed -n 1p <<< "${credentials}")" \ |
||||
|
BW_LIVE_PASSWORD="$(sed -n 2p <<< "${credentials}")"; then |
||||
|
results+=("${vector}: pass") |
||||
|
else |
||||
|
results+=("${vector}: FAIL") |
||||
|
failed=1 |
||||
|
fi |
||||
|
stop_server |
||||
|
done |
||||
|
fi |
||||
|
|
||||
|
echo |
||||
|
echo "=== Summary (logs in ${OUT_DIR})" |
||||
|
printf '%s\n' "${results[@]}" |
||||
|
exit "${failed}" |
||||
@ -0,0 +1,59 @@ |
|||||
|
"""Seeds one of the SDK's user test vectors, vault included, through vaultwarden's test seeder. |
||||
|
|
||||
|
Usage: seed_vector.py <vector.json> <seeder url> <seeder username> <seeder password> |
||||
|
|
||||
|
Prints the account's email and password, one per line. Exits 3, printing why, for a vector the tests can't log in |
||||
|
with (no master password), and 1 if the seeder refuses it. |
||||
|
""" |
||||
|
|
||||
|
import base64 |
||||
|
import json |
||||
|
import sys |
||||
|
import urllib.error |
||||
|
import urllib.request |
||||
|
import uuid |
||||
|
|
||||
|
SKIP = 3 |
||||
|
|
||||
|
|
||||
|
def main() -> int: |
||||
|
vector_file, seeder_url, username, password = sys.argv[1:5] |
||||
|
with open(vector_file, encoding="utf-8") as f: |
||||
|
vector = json.load(f) |
||||
|
|
||||
|
has_master_password = any("masterPasswordUnlock" in method for method in vector.get("unlockMethods", [])) |
||||
|
if not has_master_password or "password" not in vector.get("account", {}): |
||||
|
print("no master password") |
||||
|
return SKIP |
||||
|
|
||||
|
# Without attachments: the rotation test compares the vault before and after, and our sync gives each attachment |
||||
|
# a link with a fresh download token every time (upstream's is the same on every sync). A vector's legacy |
||||
|
# attachments, without a key of their own, would also make the SDK refuse to rotate at all. |
||||
|
for cipher in vector.get("vault", {}).get("ciphers", []): |
||||
|
cipher.get("encrypted", {}).pop("attachments", None) |
||||
|
|
||||
|
credentials = base64.b64encode(f"{username}:{password}".encode()).decode() |
||||
|
request = urllib.request.Request( |
||||
|
f"{seeder_url}/seed", |
||||
|
data=json.dumps({"template": "TestVectorUserScene", "arguments": vector}).encode(), |
||||
|
method="POST", |
||||
|
headers={ |
||||
|
"Authorization": f"Basic {credentials}", |
||||
|
"Content-Type": "application/json", |
||||
|
"X-Play-Id": f"sdk-live-tests-{uuid.uuid4()}", |
||||
|
}, |
||||
|
) |
||||
|
try: |
||||
|
with urllib.request.urlopen(request) as response: |
||||
|
result = json.load(response)["result"] |
||||
|
except urllib.error.HTTPError as e: |
||||
|
print(f"The seeder refused {vector_file}: {e.code} {e.read().decode(errors='replace')}", file=sys.stderr) |
||||
|
return 1 |
||||
|
|
||||
|
print(result["email"]) |
||||
|
print(vector["account"]["password"]) |
||||
|
return 0 |
||||
|
|
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
sys.exit(main()) |
||||
Loading…
Reference in new issue