From b0faaf25277fbbd2aea85b9c38868eb66ab45cc8 Mon Sep 17 00:00:00 2001 From: BlackDex Date: Sun, 20 Mar 2022 18:51:24 +0100 Subject: [PATCH] Several updates and fixes - Removed all `thread::sleep` and use `tokio::time::sleep` now. This solves an issue with updating to Bullseye ( Resolves #1998 ) - Updated all Debian images to Bullseye - Added MiMalloc feature and enabled it by default for Alpine based images This increases performance for the Alpine images because the default memory allocator for MUSL based binaries isn't that fast - Updated `dotenv` to `dotenvy` a maintained and updated fork - Fixed an issue with a newer jslib (not fully released yet) That version uses a different endpoint for `prelogin` Resolves #2378 ) --- .github/workflows/build.yml | 2 +- .pre-commit-config.yaml | 4 +- Cargo.lock | 141 ++++++++++++++++++-------- Cargo.toml | 11 +- docker/Dockerfile.j2 | 15 ++- docker/amd64/Dockerfile | 4 +- docker/amd64/Dockerfile.alpine | 3 +- docker/amd64/Dockerfile.buildx | 4 +- docker/amd64/Dockerfile.buildx.alpine | 3 +- docker/arm64/Dockerfile | 4 +- docker/arm64/Dockerfile.alpine | 3 +- docker/arm64/Dockerfile.buildx | 4 +- docker/arm64/Dockerfile.buildx.alpine | 3 +- docker/armv6/Dockerfile | 4 +- docker/armv6/Dockerfile.alpine | 3 +- docker/armv6/Dockerfile.buildx | 4 +- docker/armv6/Dockerfile.buildx.alpine | 3 +- docker/armv7/Dockerfile | 4 +- docker/armv7/Dockerfile.alpine | 3 +- docker/armv7/Dockerfile.buildx | 4 +- docker/armv7/Dockerfile.buildx.alpine | 3 +- src/api/core/accounts.rs | 15 +-- src/api/core/mod.rs | 2 +- src/api/identity.rs | 10 +- src/config.rs | 9 +- src/main.rs | 18 +++- src/util.rs | 13 +-- 27 files changed, 197 insertions(+), 99 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 465cf2a5..cd17230d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: include: - target-triple: x86_64-unknown-linux-gnu host-triple: x86_64-unknown-linux-gnu - features: [sqlite,mysql,postgresql] # Remember to update the `cargo test` to match the amount of features + features: [sqlite,mysql,postgresql,enable_mimalloc] # Remember to update the `cargo test` to match the amount of features channel: stable os: ubuntu-20.04 ext: "" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f18ddbf1..248e0f18 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: description: Test the package for errors. entry: cargo test language: system - args: ["--features", "sqlite,mysql,postgresql", "--"] + args: ["--features", "sqlite,mysql,postgresql,enable_mimalloc", "--"] types: [rust] pass_filenames: false - id: cargo-clippy @@ -33,6 +33,6 @@ repos: description: Lint Rust sources entry: cargo clippy language: system - args: ["--features", "sqlite,mysql,postgresql", "--", "-D", "warnings"] + args: ["--features", "sqlite,mysql,postgresql,enable_mimalloc", "--", "-D", "warnings"] types: [rust] pass_filenames: false diff --git a/Cargo.lock b/Cargo.lock index a8b21b91..526b4a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,9 +101,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" dependencies = [ "async-stream-impl", "futures-core", @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ "proc-macro2", "quote", @@ -480,9 +480,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] @@ -509,9 +509,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -603,9 +603,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0834a35a3fce649144119e18da2a4d8ed12ef3862f47183fd46f625d072d96c" +checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c" dependencies = [ "cfg-if 1.0.0", "num_cpus", @@ -727,6 +727,26 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + [[package]] name = "discard" version = "1.0.4" @@ -734,10 +754,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" [[package]] -name = "dotenv" -version = "0.15.0" +name = "dotenvy" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +checksum = "7e851a83c30366fd01d75b913588e95e74a1705c1ecc5d58b1f8e1a6d556525f" +dependencies = [ + "dirs", +] [[package]] name = "either" @@ -1424,7 +1447,7 @@ dependencies = [ "idna 0.2.3", "mime", "native-tls", - "nom 7.1.0", + "nom 7.1.1", "once_cell", "quoted_printable", "regex", @@ -1434,9 +1457,18 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.119" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7705fc40f6ed493f73584abbb324e74f96b358ff60dfe5659a0f8fc12c590a69" +dependencies = [ + "cc", +] [[package]] name = "libsqlite3-sys" @@ -1575,6 +1607,15 @@ dependencies = [ "syn", ] +[[package]] +name = "mimalloc" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0dfa131390c2f6bdb3242f65ff271fcdaca5ff7b6c08f28398be7f2280e3926" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.16" @@ -1618,14 +1659,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" dependencies = [ "libc", "log", "miow 0.3.7", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -1751,13 +1793,12 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check", ] [[package]] @@ -1828,9 +1869,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" dependencies = [ "libc", ] @@ -1884,9 +1925,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.17.0+1.1.1m" +version = "111.18.0+1.1.1n" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d6a336abd10814198f66e2a91ccd7336611f30334119ca8ce300536666fcf4" +checksum = "7897a926e1e8d00219127dc020130eca4292e5ca666dd592480d72c3eca2ff6c" dependencies = [ "cc", ] @@ -2279,9 +2320,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" dependencies = [ "proc-macro2", ] @@ -2404,9 +2445,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.2.0" +version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "929f54e29691d4e6a9cc558479de70db7aa3d98cd6fe7ab86d7507aa2886b9d2" +checksum = "738bc47119e3eeccc7e94c4a506901aea5e7b4944ecd0829cbebf4af04ceda12" dependencies = [ "bitflags", ] @@ -2429,6 +2470,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" +dependencies = [ + "getrandom 0.2.5", + "redox_syscall", + "thiserror", +] + [[package]] name = "ref-cast" version = "1.0.6" @@ -2486,9 +2538,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" dependencies = [ "async-compression", "base64 0.13.0", @@ -2524,7 +2576,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.7.0", + "winreg 0.10.1", ] [[package]] @@ -3092,9 +3144,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" dependencies = [ "proc-macro2", "quote", @@ -3256,7 +3308,7 @@ dependencies = [ "bytes 1.1.0", "libc", "memchr", - "mio 0.8.0", + "mio 0.8.2", "num_cpus", "once_cell", "parking_lot 0.12.0", @@ -3290,9 +3342,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" +checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" dependencies = [ "rustls", "tokio", @@ -3632,7 +3684,7 @@ dependencies = [ "data-url", "diesel", "diesel_migrations", - "dotenv", + "dotenvy", "fern", "futures", "governor", @@ -3644,6 +3696,7 @@ dependencies = [ "lettre", "libsqlite3-sys", "log", + "mimalloc", "num-derive", "num-traits", "once_cell", @@ -3717,6 +3770,12 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.79" @@ -3800,7 +3859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90b266eccb4b32595876f5c73ea443b0516da0b1df72ca07bc08ed9ba7f96ec1" dependencies = [ "base64 0.13.0", - "nom 7.1.0", + "nom 7.1.1", "openssl", "rand 0.8.5", "serde", @@ -3925,9 +3984,9 @@ dependencies = [ [[package]] name = "winreg" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi 0.3.9", ] diff --git a/Cargo.toml b/Cargo.toml index 98563c73..45057c63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,9 @@ postgresql = ["diesel/postgres", "diesel_migrations/postgres"] sqlite = ["diesel/sqlite", "diesel_migrations/sqlite", "libsqlite3-sys"] # Enable to use a vendored and statically linked openssl vendored_openssl = ["openssl/vendored"] +# Enable MiMalloc memory allocator to replace the default malloc +# This can improve performance for Alpine builds +enable_mimalloc = ["mimalloc"] # Enable unstable features, requires nightly # Currently only used to enable rusts official ip support @@ -38,7 +41,7 @@ tracing = { version = "0.1.32", features = ["log"] } # Needed to have lettre and backtrace = "0.3.64" # Logging panics to logfile instead stderr only # A `dotenv` implementation for Rust -dotenv = { version = "0.15.0", default-features = false } +dotenvy = { version = "0.15.1", default-features = false } # Lazy initialization once_cell = "1.10.0" @@ -113,7 +116,7 @@ percent-encoding = "2.1.0" # URL encoding library used for URL's in the emails handlebars = { version = "4.2.2", features = ["dir_source"] } # HTTP client -reqwest = { version = "0.11.9", features = ["stream", "json", "gzip", "brotli", "socks", "cookies", "trust-dns"] } +reqwest = { version = "0.11.10", features = ["stream", "json", "gzip", "brotli", "socks", "cookies", "trust-dns"] } # For favicon extraction from main website html5gum = "0.4.0" @@ -139,6 +142,10 @@ governor = "0.4.2" # Capture CTRL+C ctrlc = { version = "3.2.1", features = ["termination"] } +# Allow overriding the default memory allocator +# Mainly used for the musl builds, since the default musl malloc is very slow +mimalloc = { version = "0.1.28", features = ["secure"], default-features = false, optional = true } + [patch.crates-io] rocket = { git = 'https://github.com/SergioBenitez/Rocket', rev = 'ae0ccf43f11be5c00bb9cd49996c8bb06a7e1651' } diff --git a/docker/Dockerfile.j2 b/docker/Dockerfile.j2 index d5d6da9a..2e7fd71d 100644 --- a/docker/Dockerfile.j2 +++ b/docker/Dockerfile.j2 @@ -3,7 +3,7 @@ # This file was generated using a Jinja2 template. # Please make your changes in `Dockerfile.j2` and then `make` the individual Dockerfiles. -{% set build_stage_base_image = "rust:1.58-buster" %} +{% set build_stage_base_image = "rust:1.59-bullseye" %} {% if "alpine" in target_file %} {% if "amd64" in target_file %} {% set build_stage_base_image = "blackdex/rust-musl:x86_64-musl-stable" %} @@ -23,19 +23,19 @@ {% set package_arch_target = "aarch64-unknown-linux-musl" %} {% endif %} {% elif "amd64" in target_file %} -{% set runtime_stage_base_image = "debian:buster-slim" %} +{% set runtime_stage_base_image = "debian:bullseye-slim" %} {% elif "arm64" in target_file %} -{% set runtime_stage_base_image = "balenalib/aarch64-debian:buster" %} +{% set runtime_stage_base_image = "balenalib/aarch64-debian:bullseye" %} {% set package_arch_name = "arm64" %} {% set package_arch_target = "aarch64-unknown-linux-gnu" %} {% set package_cross_compiler = "aarch64-linux-gnu" %} {% elif "armv6" in target_file %} -{% set runtime_stage_base_image = "balenalib/rpi-debian:buster" %} +{% set runtime_stage_base_image = "balenalib/rpi-debian:bullseye" %} {% set package_arch_name = "armel" %} {% set package_arch_target = "arm-unknown-linux-gnueabi" %} {% set package_cross_compiler = "arm-linux-gnueabi" %} {% elif "armv7" in target_file %} -{% set runtime_stage_base_image = "balenalib/armv7hf-debian:buster" %} +{% set runtime_stage_base_image = "balenalib/armv7hf-debian:bullseye" %} {% set package_arch_name = "armhf" %} {% set package_arch_target = "armv7-unknown-linux-gnueabihf" %} {% set package_cross_compiler = "arm-linux-gnueabihf" %} @@ -163,7 +163,12 @@ RUN {{ mount_rust_cache -}} rustup target add {{ package_arch_target }} {% endif %} # Configure the DB ARG as late as possible to not invalidate the cached layers above +{% if "alpine" in target_file %} +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc +{% else %} ARG DB=sqlite,mysql,postgresql +{% endif %} # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/amd64/Dockerfile b/docker/amd64/Dockerfile index ae837d1e..cd44d38c 100644 --- a/docker/amd64/Dockerfile +++ b/docker/amd64/Dockerfile @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -87,7 +87,7 @@ RUN cargo build --features ${DB} --release ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM debian:buster-slim +FROM debian:bullseye-slim ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/amd64/Dockerfile.alpine b/docker/amd64/Dockerfile.alpine index 659b1580..4109c1e8 100644 --- a/docker/amd64/Dockerfile.alpine +++ b/docker/amd64/Dockerfile.alpine @@ -58,7 +58,8 @@ COPY ./build.rs ./build.rs RUN rustup target add x86_64-unknown-linux-musl # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/amd64/Dockerfile.buildx b/docker/amd64/Dockerfile.buildx index 90af9c73..54285af8 100644 --- a/docker/amd64/Dockerfile.buildx +++ b/docker/amd64/Dockerfile.buildx @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -87,7 +87,7 @@ RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/. ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM debian:buster-slim +FROM debian:bullseye-slim ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/amd64/Dockerfile.buildx.alpine b/docker/amd64/Dockerfile.buildx.alpine index cb7f5b7e..47d15659 100644 --- a/docker/amd64/Dockerfile.buildx.alpine +++ b/docker/amd64/Dockerfile.buildx.alpine @@ -58,7 +58,8 @@ COPY ./build.rs ./build.rs RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/.cargo/registry rustup target add x86_64-unknown-linux-musl # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/arm64/Dockerfile b/docker/arm64/Dockerfile index b2806d87..b462ed74 100644 --- a/docker/arm64/Dockerfile +++ b/docker/arm64/Dockerfile @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -107,7 +107,7 @@ RUN cargo build --features ${DB} --release --target=aarch64-unknown-linux-gnu ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM balenalib/aarch64-debian:buster +FROM balenalib/aarch64-debian:bullseye ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/arm64/Dockerfile.alpine b/docker/arm64/Dockerfile.alpine index 2a5cb9b1..09921643 100644 --- a/docker/arm64/Dockerfile.alpine +++ b/docker/arm64/Dockerfile.alpine @@ -58,7 +58,8 @@ COPY ./build.rs ./build.rs RUN rustup target add aarch64-unknown-linux-musl # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/arm64/Dockerfile.buildx b/docker/arm64/Dockerfile.buildx index d14680fe..c0240c36 100644 --- a/docker/arm64/Dockerfile.buildx +++ b/docker/arm64/Dockerfile.buildx @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -107,7 +107,7 @@ RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/. ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM balenalib/aarch64-debian:buster +FROM balenalib/aarch64-debian:bullseye ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/arm64/Dockerfile.buildx.alpine b/docker/arm64/Dockerfile.buildx.alpine index 1a5797d6..8f08cdf8 100644 --- a/docker/arm64/Dockerfile.buildx.alpine +++ b/docker/arm64/Dockerfile.buildx.alpine @@ -58,7 +58,8 @@ COPY ./build.rs ./build.rs RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/.cargo/registry rustup target add aarch64-unknown-linux-musl # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/armv6/Dockerfile b/docker/armv6/Dockerfile index 31861667..e727eed8 100644 --- a/docker/armv6/Dockerfile +++ b/docker/armv6/Dockerfile @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -107,7 +107,7 @@ RUN cargo build --features ${DB} --release --target=arm-unknown-linux-gnueabi ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM balenalib/rpi-debian:buster +FROM balenalib/rpi-debian:bullseye ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/armv6/Dockerfile.alpine b/docker/armv6/Dockerfile.alpine index 3dd50e8a..0b062e72 100644 --- a/docker/armv6/Dockerfile.alpine +++ b/docker/armv6/Dockerfile.alpine @@ -58,7 +58,8 @@ COPY ./build.rs ./build.rs RUN rustup target add arm-unknown-linux-musleabi # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/armv6/Dockerfile.buildx b/docker/armv6/Dockerfile.buildx index 76d4f27d..658a0a96 100644 --- a/docker/armv6/Dockerfile.buildx +++ b/docker/armv6/Dockerfile.buildx @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -107,7 +107,7 @@ RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/. ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM balenalib/rpi-debian:buster +FROM balenalib/rpi-debian:bullseye ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/armv6/Dockerfile.buildx.alpine b/docker/armv6/Dockerfile.buildx.alpine index 391090cf..b0b5ec1b 100644 --- a/docker/armv6/Dockerfile.buildx.alpine +++ b/docker/armv6/Dockerfile.buildx.alpine @@ -58,7 +58,8 @@ COPY ./build.rs ./build.rs RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/.cargo/registry rustup target add arm-unknown-linux-musleabi # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/armv7/Dockerfile b/docker/armv7/Dockerfile index 970448f2..0986e02e 100644 --- a/docker/armv7/Dockerfile +++ b/docker/armv7/Dockerfile @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -107,7 +107,7 @@ RUN cargo build --features ${DB} --release --target=armv7-unknown-linux-gnueabih ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM balenalib/armv7hf-debian:buster +FROM balenalib/armv7hf-debian:bullseye ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/armv7/Dockerfile.alpine b/docker/armv7/Dockerfile.alpine index 8bd28535..2ac9bfa0 100644 --- a/docker/armv7/Dockerfile.alpine +++ b/docker/armv7/Dockerfile.alpine @@ -59,7 +59,8 @@ COPY ./build.rs ./build.rs RUN rustup target add armv7-unknown-linux-musleabihf # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/docker/armv7/Dockerfile.buildx b/docker/armv7/Dockerfile.buildx index a5207747..3e31da87 100644 --- a/docker/armv7/Dockerfile.buildx +++ b/docker/armv7/Dockerfile.buildx @@ -27,7 +27,7 @@ FROM vaultwarden/web-vault@sha256:4412f0790fc1b8c7c86fc07f7761cd37c554802738629e80c7aa35d8bf754f9f as vault ########################## BUILD IMAGE ########################## -FROM rust:1.58-buster as build +FROM rust:1.59-bullseye as build @@ -107,7 +107,7 @@ RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/. ######################## RUNTIME IMAGE ######################## # Create a new stage with a minimal image # because we already have a binary built -FROM balenalib/armv7hf-debian:buster +FROM balenalib/armv7hf-debian:bullseye ENV ROCKET_PROFILE="release" \ ROCKET_ADDRESS=0.0.0.0 \ diff --git a/docker/armv7/Dockerfile.buildx.alpine b/docker/armv7/Dockerfile.buildx.alpine index e5e211f5..27810277 100644 --- a/docker/armv7/Dockerfile.buildx.alpine +++ b/docker/armv7/Dockerfile.buildx.alpine @@ -59,7 +59,8 @@ COPY ./build.rs ./build.rs RUN --mount=type=cache,target=/root/.cargo/git --mount=type=cache,target=/root/.cargo/registry rustup target add armv7-unknown-linux-musleabihf # Configure the DB ARG as late as possible to not invalidate the cached layers above -ARG DB=sqlite,mysql,postgresql +# Enable MiMalloc to improve performance on Alpine builds +ARG DB=sqlite,mysql,postgresql,enable_mimalloc # Builds your dependencies and removes the # dummy project, except the target folder diff --git a/src/api/core/accounts.rs b/src/api/core/accounts.rs index 1408ac6f..d0a9b37a 100644 --- a/src/api/core/accounts.rs +++ b/src/api/core/accounts.rs @@ -599,12 +599,11 @@ async fn password_hint(data: JsonUpcase, conn: DbConn) -> Empt // There is still a timing side channel here in that the code // paths that send mail take noticeably longer than ones that // don't. Add a randomized sleep to mitigate this somewhat. - use rand::{thread_rng, Rng}; - let mut rng = thread_rng(); - let base = 1000; + use rand::{rngs::SmallRng, Rng, SeedableRng}; + let mut rng = SmallRng::from_entropy(); let delta: i32 = 100; - let sleep_ms = (base + rng.gen_range(-delta..=delta)) as u64; - std::thread::sleep(std::time::Duration::from_millis(sleep_ms)); + let sleep_ms = (1_000 + rng.gen_range(-delta..=delta)) as u64; + tokio::time::sleep(tokio::time::Duration::from_millis(sleep_ms)).await; Ok(()) } else { err!(NO_HINT); @@ -626,12 +625,16 @@ async fn password_hint(data: JsonUpcase, conn: DbConn) -> Empt #[derive(Deserialize)] #[allow(non_snake_case)] -struct PreloginData { +pub struct PreloginData { Email: String, } #[post("/accounts/prelogin", data = "")] async fn prelogin(data: JsonUpcase, conn: DbConn) -> Json { + _prelogin(data, conn).await +} + +pub async fn _prelogin(data: JsonUpcase, conn: DbConn) -> Json { let data: PreloginData = data.into_inner().data; let (kdf_type, kdf_iter) = match User::find_by_mail(&data.Email, &conn).await { diff --git a/src/api/core/mod.rs b/src/api/core/mod.rs index e34343d3..381c654d 100644 --- a/src/api/core/mod.rs +++ b/src/api/core/mod.rs @@ -1,4 +1,4 @@ -mod accounts; +pub mod accounts; mod ciphers; mod emergency_access; mod folders; diff --git a/src/api/identity.rs b/src/api/identity.rs index 2ed889bb..fc7ffcec 100644 --- a/src/api/identity.rs +++ b/src/api/identity.rs @@ -9,8 +9,9 @@ use serde_json::Value; use crate::{ api::{ + core::accounts::{PreloginData, _prelogin}, core::two_factor::{duo, email, email::EmailTokenData, yubikey}, - ApiResult, EmptyResult, JsonResult, + ApiResult, EmptyResult, JsonResult, JsonUpcase, }, auth::ClientIp, db::{models::*, DbConn}, @@ -19,7 +20,7 @@ use crate::{ }; pub fn routes() -> Vec { - routes![login] + routes![login, prelogin] } #[post("/connect/token", data = "")] @@ -449,6 +450,11 @@ async fn _json_err_twofactor(providers: &[i32], user_uuid: &str, conn: &DbConn) Ok(result) } +#[post("/accounts/prelogin", data = "")] +async fn prelogin(data: JsonUpcase, conn: DbConn) -> Json { + _prelogin(data, conn).await +} + // https://github.com/bitwarden/jslib/blob/master/common/src/models/request/tokenRequest.ts // https://github.com/bitwarden/mobile/blob/master/src/Core/Models/Request/TokenRequest.cs #[derive(Debug, Clone, Default, FromForm)] diff --git a/src/config.rs b/src/config.rs index 59f02b74..2cef76e2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -59,13 +59,13 @@ macro_rules! make_config { impl ConfigBuilder { #[allow(clippy::field_reassign_with_default)] fn from_env() -> Self { - match dotenv::from_path(get_env("ENV_FILE").unwrap_or_else(|| String::from(".env"))) { + match dotenvy::from_path(get_env("ENV_FILE").unwrap_or_else(|| String::from(".env"))) { Ok(_) => (), Err(e) => match e { - dotenv::Error::LineParse(msg, pos) => { + dotenvy::Error::LineParse(msg, pos) => { panic!("Error loading the .env file:\nNear {:?} on position {}\nPlease fix and restart!\n", msg, pos); }, - dotenv::Error::Io(ioerr) => match ioerr.kind() { + dotenvy::Error::Io(ioerr) => match ioerr.kind() { std::io::ErrorKind::NotFound => { println!("[INFO] No .env file found.\n"); }, @@ -955,7 +955,8 @@ impl Config { handle.shutdown().ok(); } // Wait a bit before stopping the web server - std::thread::sleep(std::time::Duration::from_secs(1)); + tokio::runtime::Handle::current() + .block_on(async move { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await }); if let Some(handle) = c.rocket_shutdown_handle.clone() { handle.notify(); } diff --git a/src/main.rs b/src/main.rs index 08ac9d7a..59ac7eef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,13 @@ // If you go above 128 it will cause rust-analyzer to fail, #![recursion_limit = "87"] +// When enabled use MiMalloc as malloc instead of the default malloc +#[cfg(feature = "enable_mimalloc")] +use mimalloc::MiMalloc; +#[cfg(feature = "enable_mimalloc")] +#[cfg_attr(feature = "enable_mimalloc", global_allocator)] +static GLOBAL: MiMalloc = MiMalloc; + #[macro_use] extern crate rocket; #[macro_use] @@ -28,7 +35,6 @@ use std::{ process::exit, str::FromStr, thread, - time::Duration, }; #[macro_use] @@ -71,7 +77,7 @@ async fn main() -> Result<(), Error> { create_dir(&CONFIG.sends_folder(), "sends folder"); create_dir(&CONFIG.attachments_folder(), "attachments folder"); - let pool = create_db_pool(); + let pool = create_db_pool().await; schedule_jobs(pool.clone()).await; crate::db::models::TwoFactor::migrate_u2f_to_webauthn(&pool.get().await.unwrap()).await.unwrap(); @@ -315,8 +321,8 @@ fn check_web_vault() { } } -fn create_db_pool() -> db::DbPool { - match util::retry_db(db::DbPool::from_config, CONFIG.db_connection_retries()) { +async fn create_db_pool() -> db::DbPool { + match util::retry_db(db::DbPool::from_config, CONFIG.db_connection_retries()).await { Ok(p) => p, Err(e) => { error!("Error creating database pool: {:?}", e); @@ -430,7 +436,9 @@ async fn schedule_jobs(pool: db::DbPool) { // tick, the one that was added earlier will run first. loop { sched.tick(); - thread::sleep(Duration::from_millis(CONFIG.job_poll_interval_ms())); + runtime.block_on(async move { + tokio::time::sleep(tokio::time::Duration::from_millis(CONFIG.job_poll_interval_ms())).await + }); } }) .expect("Error spawning job scheduler thread"); diff --git a/src/util.rs b/src/util.rs index de61a354..ae2639fb 100644 --- a/src/util.rs +++ b/src/util.rs @@ -11,8 +11,10 @@ use rocket::{ Data, Orbit, Request, Response, Rocket, }; -use std::thread::sleep; -use std::time::Duration; +use tokio::{ + runtime::Handle, + time::{sleep, Duration}, +}; use crate::CONFIG; @@ -581,14 +583,13 @@ where if tries >= max_tries { return err; } - - sleep(Duration::from_millis(500)); + Handle::current().block_on(async move { sleep(Duration::from_millis(500)).await }); } } } } -pub fn retry_db(func: F, max_tries: u32) -> Result +pub async fn retry_db(func: F, max_tries: u32) -> Result where F: Fn() -> Result, E: std::error::Error, @@ -607,7 +608,7 @@ where warn!("Can't connect to database, retrying: {:?}", e); - sleep(Duration::from_millis(1_000)); + sleep(Duration::from_millis(1_000)).await; } } }