From e1ae0d46da89e0786b3a4e22ecbb55f7b5aa1274 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 15 Feb 2024 12:53:57 -0800 Subject: [PATCH] Add an extensions API to the collaboration server (#7807) This PR adds a REST API to the collab server for searching and downloading extensions. Previously, we had implemented this API in zed.dev directly, but this implementation is better, because we use the collab database to store the download counts for extensions. Release Notes: - N/A --------- Co-authored-by: Marshall Bowers Co-authored-by: Marshall Co-authored-by: Conrad --- .github/workflows/ci.yml | 3 + .github/workflows/deploy_collab.yml | 34 +- .gitignore | 5 +- Cargo.lock | 745 ++++++++++++++++-- Cargo.toml | 187 ++--- Procfile | 3 +- crates/collab/.env.toml | 5 + crates/collab/Cargo.toml | 21 +- crates/collab/k8s/collab.template.yml | 25 + .../20221109000000_test_schema.sql | 22 + .../20240214102900_add_extensions.sql | 22 + crates/collab/src/api.rs | 5 + crates/collab/src/api/extensions.rs | 237 ++++++ crates/collab/src/db.rs | 55 +- crates/collab/src/db/ids.rs | 1 + crates/collab/src/db/queries.rs | 1 + crates/collab/src/db/queries/extensions.rs | 205 +++++ crates/collab/src/db/tables.rs | 2 + crates/collab/src/db/tables/extension.rs | 27 + .../collab/src/db/tables/extension_version.rs | 36 + crates/collab/src/db/tests.rs | 1 + crates/collab/src/db/tests/extension_tests.rs | 219 +++++ crates/collab/src/env.rs | 3 +- crates/collab/src/lib.rs | 45 ++ crates/collab/src/main.rs | 7 +- crates/collab/src/tests/test_server.rs | 6 + script/bootstrap | 4 + script/seed-db | 1 - 28 files changed, 1754 insertions(+), 173 deletions(-) create mode 100644 crates/collab/migrations/20240214102900_add_extensions.sql create mode 100644 crates/collab/src/api/extensions.rs create mode 100644 crates/collab/src/db/queries/extensions.rs create mode 100644 crates/collab/src/db/tables/extension.rs create mode 100644 crates/collab/src/db/tables/extension_version.rs create mode 100644 crates/collab/src/db/tests/extension_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 138fa5808e25b07e141da0bff2b171cc414984ba..a3073029c257befd3cfed792788f86740b04ed5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: submodules: "recursive" fetch-depth: 0 + - name: Remove untracked files + run: git clean -df + - name: Set up default .cargo/config.toml run: cp ./.cargo/ci-config.toml ~/.cargo/config.toml diff --git a/.github/workflows/deploy_collab.yml b/.github/workflows/deploy_collab.yml index eca43501d49e7d822a5789980c475c5ea21f8df3..4d3ad09cbcfd53797f9c864f7ac4cc1c8c1c2675 100644 --- a/.github/workflows/deploy_collab.yml +++ b/.github/workflows/deploy_collab.yml @@ -45,8 +45,18 @@ jobs: submodules: "recursive" fetch-depth: 0 + - name: Install cargo nextest + shell: bash -euxo pipefail {0} + run: | + cargo install cargo-nextest + + - name: Limit target directory size + shell: bash -euxo pipefail {0} + run: script/clear-target-dir-if-larger-than 100 + - name: Run tests - uses: ./.github/actions/run_tests + shell: bash -euxo pipefail {0} + run: cargo nextest run --package collab --no-fail-fast publish: name: Publish collab server image @@ -90,22 +100,26 @@ jobs: - name: Sign into Kubernetes run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} - - name: Determine namespace + - name: Start rollout run: | set -eu if [[ $GITHUB_REF_NAME = "collab-production" ]]; then - echo "Deploying collab:$GITHUB_SHA to production" - echo "KUBE_NAMESPACE=production" >> $GITHUB_ENV + export ZED_KUBE_NAMESPACE=production elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then - echo "Deploying collab:$GITHUB_SHA to staging" - echo "KUBE_NAMESPACE=staging" >> $GITHUB_ENV + export ZED_KUBE_NAMESPACE=staging else echo "cowardly refusing to deploy from an unknown branch" exit 1 fi - - name: Start rollout - run: kubectl -n "$KUBE_NAMESPACE" set image deployment/collab collab=registry.digitalocean.com/zed/collab:${GITHUB_SHA} + echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE" + + source script/lib/deploy-helpers.sh + export_vars_for_environment $ZED_KUBE_NAMESPACE + + export ZED_DO_CERTIFICATE_ID=$(doctl compute certificate list --format ID --no-header) + export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}" - - name: Wait for rollout to finish - run: kubectl -n "$KUBE_NAMESPACE" rollout status deployment/collab + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/collab --watch + echo "deployed collab.template.yml to ${ZED_KUBE_NAMESPACE}" diff --git a/.gitignore b/.gitignore index 6f040dd0c5400c56650cabe7f1fe0b518821aa18..9b6df52dd10f1f083785c606fa1d138c53ec8236 100644 --- a/.gitignore +++ b/.gitignore @@ -5,12 +5,8 @@ .DS_Store /plugins/bin /script/node_modules -/styles/node_modules -/styles/src/types/zed.ts /crates/theme/schemas/theme.json -/crates/collab/static/styles.css /crates/collab/.admins.json -/vendor/bin /assets/*licenses.md **/venv .build @@ -25,3 +21,4 @@ DerivedData/ **/*.db .pytest_cache .venv +.blob_store diff --git a/Cargo.lock b/Cargo.lock index cdab9f60a1289b1e7e20995ed75d72f0f15a896e..064192a69f3b29fcbf2ae471dd534ed0faf904a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,6 +714,368 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af266887e24cd5f6d2ea7433cacd25dcd4773b7f70e488701968a7cdf51df57" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "hex", + "http 0.2.9", + "hyper", + "ring 0.17.7", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d56f287a9e65e4914bfedb5b22c056b65e4c232fca512d5509a9df36386759f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6a29eca8ea8982028a4df81883e7001e250a21d323b86418884b5345950a4b" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "http 0.2.9", + "http-body", + "percent-encoding", + "pin-project-lite 0.2.13", + "tracing", + "uuid 1.4.1", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c977e92277652aefb9a76a0fca652b26757d6845dce0d7bf4426da80f13d85b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "http-body", + "once_cell", + "percent-encoding", + "regex-lite", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d7f527c7b28af1a641f7d89f9e6a4863e8ec00f39d2b731b056fc5ec5ce829" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0be3224cd574ee8ab5fd7c32087876f25c134c27ac603fcb38669ed8d346b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3167c60d82a13bbaef569da06041644ff41e85c6377e5dad53fa2526ccfe9d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b1cbe0eee57a213039088dbdeca7be9352f24e0d72332d961e8a1cb388f82d" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac 0.12.1", + "http 0.2.9", + "http 1.0.0", + "once_cell", + "p256", + "percent-encoding", + "ring 0.17.7", + "sha2 0.10.7", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "426a5bc369ca7c8d3686439e46edc727f397a47ab3696b13f3ae8c81b3b36132" +dependencies = [ + "futures-util", + "pin-project-lite 0.2.13", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee554133eca2611b66d23548e48f9b44713befdb025ab76bc00185b878397a1" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes 1.5.0", + "crc32c", + "crc32fast", + "hex", + "http 0.2.9", + "http-body", + "md-5", + "pin-project-lite 0.2.13", + "sha1", + "sha2 0.10.7", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +dependencies = [ + "aws-smithy-types", + "bytes 1.5.0", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d6a0619f7b67183067fa3b558f94f90753da2df8c04aeb7336d673f804b0b8" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite 0.2.13", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1c1b5186b6f5c579bf0de1bcca9dd3d946d6d51361ea1d18131f6a0b64e13ae" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c0a2ce65882e788d2cf83ff28b9b16918de0460c47bf66c5da4f6c17b4c9694" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4cb6b3afa5fc9825a75675975dcc3e21764b5476bc91dbc63df4ea3d30a576e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "h2", + "http 0.2.9", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite 0.2.13", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23165433e80c04e8c09cee66d171292ae7234bae05fa9d5636e33095eae416b2" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes 1.5.0", + "http 0.2.9", + "pin-project-lite 0.2.13", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c94a5bec34850b92c9a054dad57b95c1d47f25125f55973e19f6ad788f0381ff" +dependencies = [ + "base64-simd", + "bytes 1.5.0", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http-body", + "itoa", + "num-integer", + "pin-project-lite 0.2.13", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util 0.7.9", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16f94c9673412b7a72e3c3efec8de89081c320bf59ea12eed34c417a62ad600" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff7e122ee50ca962e9de91f5850cc37e2184b1219611eef6d44aa85929b54f6" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.9", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.5.17" @@ -727,7 +1089,7 @@ dependencies = [ "bytes 1.5.0", "futures-util", "headers", - "http", + "http 0.2.9", "http-body", "hyper", "itoa", @@ -758,7 +1120,7 @@ dependencies = [ "async-trait", "bytes 1.5.0", "futures-util", - "http", + "http 0.2.9", "http-body", "mime", "tower-layer", @@ -774,7 +1136,7 @@ dependencies = [ "axum", "bytes 1.5.0", "futures-util", - "http", + "http 0.2.9", "mime", "pin-project-lite 0.2.13", "serde", @@ -812,6 +1174,12 @@ dependencies = [ "nix 0.23.2", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -824,6 +1192,16 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -1169,6 +1547,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes 1.5.0", + "either", +] + [[package]] name = "call" version = "0.1.0" @@ -1537,6 +1925,8 @@ dependencies = [ "async-trait", "async-tungstenite", "audio", + "aws-config", + "aws-sdk-s3", "axum", "axum-extra", "base64 0.13.1", @@ -1582,6 +1972,7 @@ dependencies = [ "rpc", "scrypt", "sea-orm", + "semver", "serde", "serde_derive", "serde_json", @@ -2087,6 +2478,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crc32c" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -2149,6 +2549,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -2161,9 +2583,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" dependencies = [ "generic-array", "subtle", @@ -2286,6 +2708,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.8" @@ -2513,6 +2945,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "editor" version = "0.1.0" @@ -2579,6 +3023,26 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -2821,6 +3285,16 @@ dependencies = [ "workspace", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "file_finder" version = "0.1.0" @@ -3160,9 +3634,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -3170,9 +3644,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -3198,9 +3672,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -3219,9 +3693,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -3230,21 +3704,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures 0.1.31", "futures-channel", @@ -3546,6 +4020,17 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df00eed8d1f0db937f6be10e46e8072b0671accb504cf0f959c5c52c679f5b9" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.21" @@ -3557,7 +4042,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap 1.9.3", "slab", "tokio", @@ -3611,7 +4096,7 @@ dependencies = [ "base64 0.21.4", "bytes 1.5.0", "headers-core", - "http", + "http 0.2.9", "httpdate", "mime", "sha1", @@ -3623,7 +4108,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.9", ] [[package]] @@ -3736,6 +4221,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -3743,7 +4239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.9", "pin-project-lite 0.2.13", ] @@ -3788,7 +4284,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "httparse", "httpdate", @@ -3801,6 +4297,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.9", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -4033,7 +4545,7 @@ dependencies = [ "encoding_rs", "event-listener", "futures-lite", - "http", + "http 0.2.9", "log", "mime", "once_cell", @@ -5578,12 +6090,29 @@ dependencies = [ "workspace", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.7", +] + [[package]] name = "palette" version = "0.7.3" @@ -5688,9 +6217,9 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.2.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7" +checksum = "c1a5d4e9c205d2c1ae73b84aab6240e98218c0e72e63b50422cfb2d1ca952282" dependencies = [ "base64ct", "rand_core 0.6.4", @@ -5902,9 +6431,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.8", + "pkcs8 0.10.2", + "spki 0.7.2", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -5913,8 +6452,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.8", + "spki 0.7.2", ] [[package]] @@ -6705,6 +7244,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -6752,7 +7297,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-tls", @@ -6793,6 +7338,17 @@ dependencies = [ "usvg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac 0.12.1", + "zeroize", +] + [[package]] name = "rgb" version = "0.8.36" @@ -6831,11 +7387,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi 0.3.9", ] +[[package]] +name = "ring" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +dependencies = [ + "cc", + "getrandom 0.2.10", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rkyv" version = "0.7.42" @@ -6989,10 +7559,10 @@ dependencies = [ "num-iter", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.1.0", + "spki 0.7.2", "subtle", "zeroize", ] @@ -7124,15 +7694,28 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ - "ring", + "log", + "ring 0.17.7", "rustls-webpki", "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.3" @@ -7144,12 +7727,12 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.7", + "untrusted 0.9.0", ] [[package]] @@ -7302,8 +7885,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -7430,6 +8013,20 @@ dependencies = [ "workspace", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -7747,6 +8344,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.1.0" @@ -7934,6 +8541,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.2" @@ -7941,7 +8558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der", + "der 0.7.8", ] [[package]] @@ -8301,9 +8918,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "sum_tree" @@ -8913,6 +9530,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -9004,7 +9631,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-timeout", @@ -9052,7 +9679,7 @@ dependencies = [ "bytes 1.5.0", "futures-core", "futures-util", - "http", + "http 0.2.9", "http-body", "http-range-header", "pin-project-lite 0.2.13", @@ -9624,7 +10251,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.9", "httparse", "log", "native-tls", @@ -9644,7 +10271,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.9", "httparse", "log", "rand 0.8.5", @@ -9786,6 +10413,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.1" @@ -10000,6 +10633,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index 170a58ea7e1fd18956e39616229f80d31bc27390..b155bf2232946430a1de4e50ea60a8141419a213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,91 +1,91 @@ [workspace] members = [ - "crates/activity_indicator", - "crates/ai", - "crates/assets", - "crates/assistant", - "crates/audio", - "crates/auto_update", - "crates/breadcrumbs", - "crates/call", - "crates/channel", - "crates/cli", - "crates/client", - "crates/clock", - "crates/collab", - "crates/collab_ui", - "crates/collections", - "crates/command_palette", - "crates/copilot", - "crates/copilot_ui", - "crates/db", - "crates/diagnostics", - "crates/editor", - "crates/extension", - "crates/extensions_ui", - "crates/feature_flags", - "crates/feedback", - "crates/file_finder", - "crates/fs", - "crates/fsevent", - "crates/fuzzy", - "crates/git", - "crates/go_to_line", - "crates/gpui", - "crates/gpui_macros", - "crates/install_cli", - "crates/journal", - "crates/language", - "crates/language_selector", - "crates/language_tools", - "crates/live_kit_client", - "crates/live_kit_server", - "crates/lsp", - "crates/markdown_preview", - "crates/media", - "crates/menu", - "crates/multi_buffer", - "crates/node_runtime", - "crates/notifications", - "crates/outline", - "crates/picker", - "crates/plugin", - "crates/plugin_macros", - "crates/prettier", - "crates/project", - "crates/project_panel", - "crates/project_symbols", - "crates/quick_action_bar", - "crates/recent_projects", - "crates/refineable", - "crates/refineable/derive_refineable", - "crates/release_channel", - "crates/rich_text", - "crates/rope", - "crates/rpc", - "crates/search", - "crates/semantic_index", - "crates/settings", - "crates/snippet", - "crates/sqlez", - "crates/sqlez_macros", - "crates/story", - "crates/storybook", - "crates/sum_tree", - "crates/terminal", - "crates/terminal_view", - "crates/text", - "crates/theme", - "crates/theme_importer", - "crates/theme_selector", - "crates/ui", - "crates/util", - "crates/vcs_menu", - "crates/vim", - "crates/welcome", - "crates/workspace", - "crates/zed", - "crates/zed_actions", + "crates/activity_indicator", + "crates/ai", + "crates/assets", + "crates/assistant", + "crates/audio", + "crates/auto_update", + "crates/breadcrumbs", + "crates/call", + "crates/channel", + "crates/cli", + "crates/client", + "crates/clock", + "crates/collab", + "crates/collab_ui", + "crates/collections", + "crates/command_palette", + "crates/copilot", + "crates/copilot_ui", + "crates/db", + "crates/diagnostics", + "crates/editor", + "crates/extension", + "crates/extensions_ui", + "crates/feature_flags", + "crates/feedback", + "crates/file_finder", + "crates/fs", + "crates/fsevent", + "crates/fuzzy", + "crates/git", + "crates/go_to_line", + "crates/gpui", + "crates/gpui_macros", + "crates/install_cli", + "crates/journal", + "crates/language", + "crates/language_selector", + "crates/language_tools", + "crates/live_kit_client", + "crates/live_kit_server", + "crates/lsp", + "crates/markdown_preview", + "crates/media", + "crates/menu", + "crates/multi_buffer", + "crates/node_runtime", + "crates/notifications", + "crates/outline", + "crates/picker", + "crates/plugin", + "crates/plugin_macros", + "crates/prettier", + "crates/project", + "crates/project_panel", + "crates/project_symbols", + "crates/quick_action_bar", + "crates/recent_projects", + "crates/refineable", + "crates/refineable/derive_refineable", + "crates/release_channel", + "crates/rich_text", + "crates/rope", + "crates/rpc", + "crates/search", + "crates/semantic_index", + "crates/settings", + "crates/snippet", + "crates/sqlez", + "crates/sqlez_macros", + "crates/story", + "crates/storybook", + "crates/sum_tree", + "crates/terminal", + "crates/terminal_view", + "crates/text", + "crates/theme", + "crates/theme_importer", + "crates/theme_selector", + "crates/ui", + "crates/util", + "crates/vcs_menu", + "crates/vim", + "crates/welcome", + "crates/workspace", + "crates/zed", + "crates/zed_actions", ] default-members = ["crates/zed"] resolver = "2" @@ -191,8 +191,8 @@ globset = "0.4" indoc = "1" # We explicitly disable a http2 support in isahc. isahc = { version = "1.7.2", default-features = false, features = [ - "static-curl", - "text-decoding", + "static-curl", + "text-decoding", ] } lazy_static = "1.4.0" log = { version = "0.4.16", features = ["kv_unstable_serde"] } @@ -208,12 +208,13 @@ regex = "1.5" rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] } rust-embed = { version = "8.0", features = ["include-exclude"] } schemars = "0.8" +semver = { version = "1.0" } serde = { version = "1.0", features = ["derive", "rc"] } serde_derive = { version = "1.0", features = ["deserialize_in_place"] } serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] } serde_json_lenient = { version = "0.1", features = [ - "preserve_order", - "raw_value", + "preserve_order", + "raw_value", ] } serde_repr = "0.1" smallvec = { version = "1.6", features = ["union"] } @@ -223,7 +224,11 @@ sysinfo = "0.29.10" tempfile = "3.9.0" thiserror = "1.0.29" tiktoken-rs = "0.5.7" -time = { version = "0.3", features = ["serde", "serde-well-known"] } +time = { version = "0.3", features = [ + "serde", + "serde-well-known", + "formatting", +] } toml = "0.5" tree-sitter = { version = "0.20", features = ["wasm"] } tree-sitter-astro = { git = "https://github.com/virchau13/tree-sitter-astro.git", rev = "e924787e12e8a03194f36a113290ac11d6dc10f3" } diff --git a/Procfile b/Procfile index 7bd9114dad4ec3e89c4699d0924bbf1ef1243867..288842ebd3ca3de169f36ec67a5c09f425bfbb42 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ -collab: cd crates/collab && RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run serve +collab: RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run --package=collab serve livekit: livekit-server --dev +blob_store: MINIO_ROOT_USER=the-blob-store-access-key MINIO_ROOT_PASSWORD=the-blob-store-secret-key minio server .blob_store diff --git a/crates/collab/.env.toml b/crates/collab/.env.toml index 01866012ea8a14c94835d0662318d3f8c4df67f5..7340a71cd9f7a1306f241dc585a431dc9be2d8c5 100644 --- a/crates/collab/.env.toml +++ b/crates/collab/.env.toml @@ -7,6 +7,11 @@ ZED_ENVIRONMENT = "development" LIVE_KIT_SERVER = "http://localhost:7880" LIVE_KIT_KEY = "devkey" LIVE_KIT_SECRET = "secret" +BLOB_STORE_ACCESS_KEY = "the-blob-store-access-key" +BLOB_STORE_SECRET_KEY = "the-blob-store-secret-key" +BLOB_STORE_BUCKET = "the-extensions-bucket" +BLOB_STORE_URL = "http://127.0.0.1:9000" +BLOB_STORE_REGION = "the-region" # RUST_LOG=info # LOG_JSON=true diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index 02b2e5eb4896475e567cfe51dcc9af1932ce8778..6c9e07f3be25272fac26b5cef050da740b020c0c 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -15,9 +15,11 @@ name = "seed" required-features = ["seed-support"] [dependencies] +axum = { version = "0.5", features = ["json", "headers", "ws"] } anyhow.workspace = true +aws-config = { version = "1.1.5" } +aws-sdk-s3 = { version = "1.15.0" } async-tungstenite = "0.16" -axum = { version = "0.5", features = ["json", "headers", "ws"] } axum-extra = { version = "0.3", features = ["erased-json"] } base64 = "0.13" chrono.workspace = true @@ -40,13 +42,26 @@ rand.workspace = true reqwest = { version = "0.11", features = ["json"], optional = true } rpc.workspace = true scrypt = "0.7" -sea-orm = { version = "0.12.x", features = ["sqlx-postgres", "postgres-array", "runtime-tokio-rustls", "with-uuid"] } +sea-orm = { version = "0.12.x", features = [ + "sqlx-postgres", + "postgres-array", + "runtime-tokio-rustls", + "with-uuid", +] } +semver.workspace = true serde.workspace = true serde_derive.workspace = true serde_json.workspace = true sha-1 = "0.9" smallvec.workspace = true -sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "any"] } +sqlx = { version = "0.7", features = [ + "runtime-tokio-rustls", + "postgres", + "json", + "time", + "uuid", + "any", +] } text.workspace = true time.workspace = true tokio = { version = "1", features = ["full"] } diff --git a/crates/collab/k8s/collab.template.yml b/crates/collab/k8s/collab.template.yml index 120e5f592f62bfebd1a8c4f6b95d8c665f5c5577..9ff7cee9e1f3c39c300a161cd1711e762c4a188a 100644 --- a/crates/collab/k8s/collab.template.yml +++ b/crates/collab/k8s/collab.template.yml @@ -105,6 +105,31 @@ spec: secretKeyRef: name: livekit key: secret + - name: BLOB_STORE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: blob-store + key: access_key + - name: BLOB_STORE_SECRET_KEY + valueFrom: + secretKeyRef: + name: blob-store + key: secret_key + - name: BLOB_STORE_URL + valueFrom: + secretKeyRef: + name: blob-store + key: url + - name: BLOB_STORE_REGION + valueFrom: + secretKeyRef: + name: blob-store + key: region + - name: BLOB_STORE_BUCKET + valueFrom: + secretKeyRef: + name: blob-store + key: bucket - name: INVITE_LINK_PREFIX value: ${INVITE_LINK_PREFIX} - name: RUST_BACKTRACE diff --git a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql index 1026fdea0d96ce96d9dffa5d3607cfb2c61bff66..fef10f987efc57f6b31caa9190d4106c0565fe5e 100644 --- a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql +++ b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql @@ -353,3 +353,25 @@ CREATE TABLE contributors ( signed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (user_id) ); + +CREATE TABLE extensions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_id TEXT NOT NULL, + name TEXT NOT NULL, + latest_version TEXT NOT NULL, + total_download_count INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE extension_versions ( + extension_id INTEGER REFERENCES extensions(id), + version TEXT NOT NULL, + published_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + authors TEXT NOT NULL, + repository TEXT NOT NULL, + description TEXT NOT NULL, + download_count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (extension_id, version) +); + +CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id"); +CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count"); diff --git a/crates/collab/migrations/20240214102900_add_extensions.sql b/crates/collab/migrations/20240214102900_add_extensions.sql new file mode 100644 index 0000000000000000000000000000000000000000..b32094036d6a8993a8dbc6dc2407dec0e53aea47 --- /dev/null +++ b/crates/collab/migrations/20240214102900_add_extensions.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS extensions ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + external_id TEXT NOT NULL, + latest_version TEXT NOT NULL, + total_download_count BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS extension_versions ( + extension_id INTEGER REFERENCES extensions(id), + version TEXT NOT NULL, + published_at TIMESTAMP NOT NULL DEFAULT now(), + authors TEXT NOT NULL, + repository TEXT NOT NULL, + description TEXT NOT NULL, + download_count BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY(extension_id, version) +); + +CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id"); +CREATE INDEX "trigram_index_extensions_name" ON "extensions" USING GIN(name gin_trgm_ops); +CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count"); diff --git a/crates/collab/src/api.rs b/crates/collab/src/api.rs index 59d176b047b508d9748ba44e4aad700e60a1f58a..44d6fc3eb5347804cf59204699b0d93f50004c4f 100644 --- a/crates/collab/src/api.rs +++ b/crates/collab/src/api.rs @@ -1,3 +1,5 @@ +mod extensions; + use crate::{ auth, db::{ContributorSelector, User, UserId}, @@ -20,6 +22,8 @@ use std::sync::Arc; use tower::ServiceBuilder; use tracing::instrument; +pub use extensions::fetch_extensions_from_blob_store_periodically; + pub fn routes(rpc_server: Arc, state: Arc) -> Router { Router::new() .route("/user", get(get_authenticated_user)) @@ -28,6 +32,7 @@ pub fn routes(rpc_server: Arc, state: Arc) -> Router Router { + Router::new() + .route("/extensions", get(get_extensions)) + .route( + "/extensions/:extension_id/:version/download", + get(download_extension), + ) +} + +#[derive(Debug, Deserialize)] +struct GetExtensionsParams { + filter: Option, +} + +#[derive(Debug, Deserialize)] +struct DownloadExtensionParams { + extension_id: String, + version: String, +} + +#[derive(Debug, Serialize)] +struct GetExtensionsResponse { + pub data: Vec, +} + +#[derive(Deserialize)] +struct ExtensionManifest { + name: String, + version: String, + description: Option, + authors: Vec, + repository: String, +} + +async fn get_extensions( + Extension(app): Extension>, + Query(params): Query, +) -> Result> { + let extensions = app.db.get_extensions(params.filter.as_deref(), 30).await?; + Ok(Json(GetExtensionsResponse { data: extensions })) +} + +async fn download_extension( + Extension(app): Extension>, + Path(params): Path, +) -> Result { + let Some((blob_store_client, bucket)) = app + .blob_store_client + .clone() + .zip(app.config.blob_store_bucket.clone()) + else { + Err(Error::Http( + StatusCode::NOT_IMPLEMENTED, + "not supported".into(), + ))? + }; + + let DownloadExtensionParams { + extension_id, + version, + } = params; + + let version_exists = app + .db + .record_extension_download(&extension_id, &version) + .await?; + + if !version_exists { + Err(Error::Http( + StatusCode::NOT_FOUND, + "unknown extension version".into(), + ))?; + } + + let url = blob_store_client + .get_object() + .bucket(bucket) + .key(format!( + "extensions/{extension_id}/{version}/archive.tar.gz" + )) + .presigned(PresigningConfig::expires_in(EXTENSION_DOWNLOAD_URL_LIFETIME).unwrap()) + .await + .map_err(|e| anyhow!("failed to create presigned extension download url {e}"))?; + + Ok(Redirect::temporary(url.uri())) +} + +const EXTENSION_FETCH_INTERVAL: Duration = Duration::from_secs(5 * 60); +const EXTENSION_DOWNLOAD_URL_LIFETIME: Duration = Duration::from_secs(3 * 60); + +pub fn fetch_extensions_from_blob_store_periodically(app_state: Arc, executor: Executor) { + let Some(blob_store_client) = app_state.blob_store_client.clone() else { + log::info!("no blob store client"); + return; + }; + let Some(blob_store_bucket) = app_state.config.blob_store_bucket.clone() else { + log::info!("no blob store bucket"); + return; + }; + + executor.spawn_detached({ + let executor = executor.clone(); + async move { + loop { + fetch_extensions_from_blob_store( + &blob_store_client, + &blob_store_bucket, + &app_state, + ) + .await + .log_err(); + executor.sleep(EXTENSION_FETCH_INTERVAL).await; + } + } + }); +} + +async fn fetch_extensions_from_blob_store( + blob_store_client: &aws_sdk_s3::Client, + blob_store_bucket: &String, + app_state: &Arc, +) -> anyhow::Result<()> { + let list = blob_store_client + .list_objects() + .bucket(blob_store_bucket) + .prefix("extensions/") + .send() + .await?; + + let objects = list + .contents + .ok_or_else(|| anyhow!("missing bucket contents"))?; + + let mut published_versions = HashMap::<&str, Vec<&str>>::default(); + for object in &objects { + let Some(key) = object.key.as_ref() else { + continue; + }; + let mut parts = key.split('/'); + let Some(_) = parts.next().filter(|part| *part == "extensions") else { + continue; + }; + let Some(extension_id) = parts.next() else { + continue; + }; + let Some(version) = parts.next() else { + continue; + }; + published_versions + .entry(extension_id) + .or_default() + .push(version); + } + + let known_versions = app_state.db.get_known_extension_versions().await?; + + let mut new_versions = HashMap::<&str, Vec>::default(); + let empty = Vec::new(); + for (extension_id, published_versions) in published_versions { + let known_versions = known_versions.get(extension_id).unwrap_or(&empty); + + for published_version in published_versions { + if known_versions + .binary_search_by_key(&published_version, String::as_str) + .is_err() + { + let object = blob_store_client + .get_object() + .bucket(blob_store_bucket) + .key(format!( + "extensions/{extension_id}/{published_version}/manifest.json" + )) + .send() + .await?; + let manifest_bytes = object + .body + .collect() + .await + .map(|data| data.into_bytes()) + .with_context(|| format!("failed to download manifest for extension {extension_id} version {published_version}"))? + .to_vec(); + let manifest = serde_json::from_slice::(&manifest_bytes) + .with_context(|| format!("invalid manifest for extension {extension_id} version {published_version}: {}", String::from_utf8_lossy(&manifest_bytes)))?; + + let published_at = object.last_modified.ok_or_else(|| anyhow!("missing last modified timestamp for extension {extension_id} version {published_version}"))?; + let published_at = + time::OffsetDateTime::from_unix_timestamp_nanos(published_at.as_nanos())?; + let published_at = PrimitiveDateTime::new(published_at.date(), published_at.time()); + + let version = semver::Version::parse(&manifest.version).with_context(|| { + format!( + "invalid version for extension {extension_id} version {published_version}" + ) + })?; + + new_versions + .entry(extension_id) + .or_default() + .push(NewExtensionVersion { + name: manifest.name, + version, + description: manifest.description.unwrap_or_default(), + authors: manifest.authors, + repository: manifest.repository, + published_at, + }); + } + } + } + + app_state + .db + .insert_extension_versions(&new_versions) + .await?; + + Ok(()) +} diff --git a/crates/collab/src/db.rs b/crates/collab/src/db.rs index 6ccea25a600b96c22e4bee351f7cf13a794b6ad2..08e502a42f2e65ba72e758011cf2cb886f1a069f 100644 --- a/crates/collab/src/db.rs +++ b/crates/collab/src/db.rs @@ -1,12 +1,8 @@ -#[cfg(test)] -pub mod tests; - -#[cfg(test)] -pub use tests::TestDb; - mod ids; mod queries; mod tables; +#[cfg(test)] +pub mod tests; use crate::{executor::Executor, Error, Result}; use anyhow::anyhow; @@ -25,7 +21,7 @@ use sea_orm::{ FromQueryResult, IntoActiveModel, IsolationLevel, JoinType, QueryOrder, QuerySelect, Statement, TransactionTrait, }; -use serde::{Deserialize, Serialize}; +use serde::{ser::Error as _, Deserialize, Serialize, Serializer}; use sqlx::{ migrate::{Migrate, Migration, MigrationSource}, Connection, @@ -40,13 +36,17 @@ use std::{ sync::Arc, time::Duration, }; -pub use tables::*; +use time::{format_description::well_known::iso8601, PrimitiveDateTime}; use tokio::sync::{Mutex, OwnedMutexGuard}; +#[cfg(test)] +pub use tests::TestDb; + pub use ids::*; pub use queries::contributors::ContributorSelector; pub use sea_orm::ConnectOptions; pub use tables::user::Model as User; +pub use tables::*; /// Database gives you a handle that lets you access the database. /// It handles pooling internally. @@ -717,3 +717,42 @@ pub struct WorktreeSettingsFile { pub path: String, pub content: String, } + +pub struct NewExtensionVersion { + pub name: String, + pub version: semver::Version, + pub description: String, + pub authors: Vec, + pub repository: String, + pub published_at: PrimitiveDateTime, +} + +#[derive(Debug, Serialize, PartialEq)] +pub struct ExtensionMetadata { + pub id: String, + pub name: String, + pub version: String, + pub authors: Vec, + pub repository: String, + #[serde(serialize_with = "serialize_iso8601")] + pub published_at: PrimitiveDateTime, + pub download_count: u64, +} + +pub fn serialize_iso8601( + datetime: &PrimitiveDateTime, + serializer: S, +) -> Result { + const SERDE_CONFIG: iso8601::EncodedConfig = iso8601::Config::DEFAULT + .set_year_is_six_digits(false) + .set_time_precision(iso8601::TimePrecision::Second { + decimal_digits: None, + }) + .encode(); + + datetime + .assume_utc() + .format(&time::format_description::well_known::Iso8601::) + .map_err(S::Error::custom)? + .serialize(serializer) +} diff --git a/crates/collab/src/db/ids.rs b/crates/collab/src/db/ids.rs index d69e19643a502a826e4c04b1716ac9865198d045..44a5db6a75140e775ba0a8527972e3251b99ca30 100644 --- a/crates/collab/src/db/ids.rs +++ b/crates/collab/src/db/ids.rs @@ -85,6 +85,7 @@ id_type!(SignupId); id_type!(UserId); id_type!(ChannelBufferCollaboratorId); id_type!(FlagId); +id_type!(ExtensionId); id_type!(NotificationId); id_type!(NotificationKindId); diff --git a/crates/collab/src/db/queries.rs b/crates/collab/src/db/queries.rs index f6bba13ede5fee59b313a602fbf25d3a3d9b3ace..7d9043f595e2890a24e213b4a03608417aad7b57 100644 --- a/crates/collab/src/db/queries.rs +++ b/crates/collab/src/db/queries.rs @@ -5,6 +5,7 @@ pub mod buffers; pub mod channels; pub mod contacts; pub mod contributors; +pub mod extensions; pub mod messages; pub mod notifications; pub mod projects; diff --git a/crates/collab/src/db/queries/extensions.rs b/crates/collab/src/db/queries/extensions.rs new file mode 100644 index 0000000000000000000000000000000000000000..78e80ea190e2542b0db4f9a085e57a138673b499 --- /dev/null +++ b/crates/collab/src/db/queries/extensions.rs @@ -0,0 +1,205 @@ +use super::*; + +impl Database { + pub async fn get_extensions( + &self, + filter: Option<&str>, + limit: usize, + ) -> Result> { + self.transaction(|tx| async move { + let mut condition = Condition::all(); + if let Some(filter) = filter { + let fuzzy_name_filter = Self::fuzzy_like_string(filter); + condition = condition.add(Expr::cust_with_expr("name ILIKE $1", fuzzy_name_filter)); + } + + let extensions = extension::Entity::find() + .filter(condition) + .order_by_desc(extension::Column::TotalDownloadCount) + .order_by_asc(extension::Column::Id) + .limit(Some(limit as u64)) + .filter( + extension::Column::LatestVersion + .into_expr() + .eq(extension_version::Column::Version.into_expr()), + ) + .inner_join(extension_version::Entity) + .select_also(extension_version::Entity) + .all(&*tx) + .await?; + + Ok(extensions + .into_iter() + .filter_map(|(extension, latest_version)| { + let version = latest_version?; + Some(ExtensionMetadata { + id: extension.external_id, + name: extension.name, + version: version.version, + authors: version + .authors + .split(',') + .map(|author| author.trim().to_string()) + .collect::>(), + repository: version.repository, + published_at: version.published_at, + download_count: extension.total_download_count as u64, + }) + }) + .collect()) + }) + .await + } + + pub async fn get_known_extension_versions<'a>(&self) -> Result>> { + self.transaction(|tx| async move { + let mut extension_external_ids_by_id = HashMap::default(); + + let mut rows = extension::Entity::find().stream(&*tx).await?; + while let Some(row) = rows.next().await { + let row = row?; + extension_external_ids_by_id.insert(row.id, row.external_id); + } + drop(rows); + + let mut known_versions_by_extension_id: HashMap> = + HashMap::default(); + let mut rows = extension_version::Entity::find().stream(&*tx).await?; + while let Some(row) = rows.next().await { + let row = row?; + + let Some(extension_id) = extension_external_ids_by_id.get(&row.extension_id) else { + continue; + }; + + let versions = known_versions_by_extension_id + .entry(extension_id.clone()) + .or_default(); + if let Err(ix) = versions.binary_search(&row.version) { + versions.insert(ix, row.version); + } + } + drop(rows); + + Ok(known_versions_by_extension_id) + }) + .await + } + + pub async fn insert_extension_versions( + &self, + versions_by_extension_id: &HashMap<&str, Vec>, + ) -> Result<()> { + self.transaction(|tx| async move { + for (external_id, versions) in versions_by_extension_id { + if versions.is_empty() { + continue; + } + + let latest_version = versions + .iter() + .max_by_key(|version| &version.version) + .unwrap(); + + let insert = extension::Entity::insert(extension::ActiveModel { + name: ActiveValue::Set(latest_version.name.clone()), + external_id: ActiveValue::Set(external_id.to_string()), + id: ActiveValue::NotSet, + latest_version: ActiveValue::Set(latest_version.version.to_string()), + total_download_count: ActiveValue::NotSet, + }) + .on_conflict( + OnConflict::columns([extension::Column::ExternalId]) + .update_column(extension::Column::ExternalId) + .to_owned(), + ); + + let extension = if tx.support_returning() { + insert.exec_with_returning(&*tx).await? + } else { + // Sqlite + insert.exec_without_returning(&*tx).await?; + extension::Entity::find() + .filter(extension::Column::ExternalId.eq(*external_id)) + .one(&*tx) + .await? + .ok_or_else(|| anyhow!("failed to insert extension"))? + }; + + extension_version::Entity::insert_many(versions.iter().map(|version| { + extension_version::ActiveModel { + extension_id: ActiveValue::Set(extension.id), + published_at: ActiveValue::Set(version.published_at), + version: ActiveValue::Set(version.version.to_string()), + authors: ActiveValue::Set(version.authors.join(", ")), + repository: ActiveValue::Set(version.repository.clone()), + description: ActiveValue::Set(version.description.clone()), + download_count: ActiveValue::NotSet, + } + })) + .on_conflict(OnConflict::new().do_nothing().to_owned()) + .exec_without_returning(&*tx) + .await?; + + if let Ok(db_version) = semver::Version::parse(&extension.latest_version) { + if db_version >= latest_version.version { + continue; + } + } + + let mut extension = extension.into_active_model(); + extension.latest_version = ActiveValue::Set(latest_version.version.to_string()); + extension.name = ActiveValue::set(latest_version.name.clone()); + extension::Entity::update(extension).exec(&*tx).await?; + } + + Ok(()) + }) + .await + } + + pub async fn record_extension_download(&self, extension: &str, version: &str) -> Result { + self.transaction(|tx| async move { + #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] + enum QueryId { + Id, + } + + let extension_id: Option = extension::Entity::find() + .filter(extension::Column::ExternalId.eq(extension)) + .select_only() + .column(extension::Column::Id) + .into_values::<_, QueryId>() + .one(&*tx) + .await?; + let Some(extension_id) = extension_id else { + return Ok(false); + }; + + extension_version::Entity::update_many() + .col_expr( + extension_version::Column::DownloadCount, + extension_version::Column::DownloadCount.into_expr().add(1), + ) + .filter( + extension_version::Column::ExtensionId + .eq(extension_id) + .and(extension_version::Column::Version.eq(version)), + ) + .exec(&*tx) + .await?; + + extension::Entity::update_many() + .col_expr( + extension::Column::TotalDownloadCount, + extension::Column::TotalDownloadCount.into_expr().add(1), + ) + .filter(extension::Column::Id.eq(extension_id)) + .exec(&*tx) + .await?; + + Ok(true) + }) + .await + } +} diff --git a/crates/collab/src/db/tables.rs b/crates/collab/src/db/tables.rs index 646447c91f6e3c56016786a5d39f81aa8f5e8eef..72d98350324e676fc605372c69a1ea62aca20f38 100644 --- a/crates/collab/src/db/tables.rs +++ b/crates/collab/src/db/tables.rs @@ -10,6 +10,8 @@ pub mod channel_message; pub mod channel_message_mention; pub mod contact; pub mod contributor; +pub mod extension; +pub mod extension_version; pub mod feature_flag; pub mod follower; pub mod language_server; diff --git a/crates/collab/src/db/tables/extension.rs b/crates/collab/src/db/tables/extension.rs new file mode 100644 index 0000000000000000000000000000000000000000..5a1462c70114feefca4c46a23259169367579ce5 --- /dev/null +++ b/crates/collab/src/db/tables/extension.rs @@ -0,0 +1,27 @@ +use crate::db::ExtensionId; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "extensions")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: ExtensionId, + pub external_id: String, + pub name: String, + pub latest_version: String, + pub total_download_count: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_one = "super::extension_version::Entity")] + LatestVersion, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LatestVersion.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/collab/src/db/tables/extension_version.rs b/crates/collab/src/db/tables/extension_version.rs new file mode 100644 index 0000000000000000000000000000000000000000..459f2296b1d28ceccf40e4b398b0b507c55efa9f --- /dev/null +++ b/crates/collab/src/db/tables/extension_version.rs @@ -0,0 +1,36 @@ +use crate::db::ExtensionId; +use sea_orm::entity::prelude::*; +use time::PrimitiveDateTime; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "extension_versions")] +pub struct Model { + #[sea_orm(primary_key)] + pub extension_id: ExtensionId, + #[sea_orm(primary_key)] + pub version: String, + pub published_at: PrimitiveDateTime, + pub authors: String, + pub repository: String, + pub description: String, + pub download_count: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::extension::Entity", + from = "Column::ExtensionId", + to = "super::extension::Column::Id" + on_condition = r#"super::extension::Column::LatestVersion.into_expr().eq(Column::Version.into_expr())"# + )] + Extension, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Extension.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/collab/src/db/tests.rs b/crates/collab/src/db/tests.rs index a85aae4fa348b13b9780c4381a6c00b7cf69afa0..7790b951b2876fce92409e3628bc495475c15953 100644 --- a/crates/collab/src/db/tests.rs +++ b/crates/collab/src/db/tests.rs @@ -2,6 +2,7 @@ mod buffer_tests; mod channel_tests; mod contributor_tests; mod db_tests; +mod extension_tests; mod feature_flag_tests; mod message_tests; diff --git a/crates/collab/src/db/tests/extension_tests.rs b/crates/collab/src/db/tests/extension_tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..8963ffcbe96342d20eec4875a1babcb0a30f1989 --- /dev/null +++ b/crates/collab/src/db/tests/extension_tests.rs @@ -0,0 +1,219 @@ +use super::Database; +use crate::{ + db::{ExtensionMetadata, NewExtensionVersion}, + test_both_dbs, +}; +use std::sync::Arc; +use time::{OffsetDateTime, PrimitiveDateTime}; + +test_both_dbs!( + test_extensions, + test_extensions_postgres, + test_extensions_sqlite +); + +async fn test_extensions(db: &Arc) { + let versions = db.get_known_extension_versions().await.unwrap(); + assert!(versions.is_empty()); + + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert!(extensions.is_empty()); + + let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap(); + let t0 = PrimitiveDateTime::new(t0.date(), t0.time()); + + db.insert_extension_versions( + &[ + ( + "ext1", + vec![ + NewExtensionVersion { + name: "Extension 1".into(), + version: semver::Version::parse("0.0.1").unwrap(), + description: "an extension".into(), + authors: vec!["max".into()], + repository: "ext1/repo".into(), + published_at: t0, + }, + NewExtensionVersion { + name: "Extension One".into(), + version: semver::Version::parse("0.0.2").unwrap(), + description: "a good extension".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + }, + ], + ), + ( + "ext2", + vec![NewExtensionVersion { + name: "Extension Two".into(), + version: semver::Version::parse("0.2.0").unwrap(), + description: "a great extension".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + }], + ), + ] + .into_iter() + .collect(), + ) + .await + .unwrap(); + + let versions = db.get_known_extension_versions().await.unwrap(); + assert_eq!( + versions, + [ + ("ext1".into(), vec!["0.0.1".into(), "0.0.2".into()]), + ("ext2".into(), vec!["0.2.0".into()]) + ] + .into_iter() + .collect() + ); + + // The latest version of each extension is returned. + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 0, + }, + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 0 + }, + ] + ); + + // Record extensions being downloaded. + for _ in 0..7 { + assert!(db.record_extension_download("ext2", "0.0.2").await.unwrap()); + } + + for _ in 0..3 { + assert!(db.record_extension_download("ext1", "0.0.1").await.unwrap()); + } + + for _ in 0..2 { + assert!(db.record_extension_download("ext1", "0.0.2").await.unwrap()); + } + + // Record download returns false if the extension does not exist. + assert!(!db + .record_extension_download("no-such-extension", "0.0.2") + .await + .unwrap()); + + // Extensions are returned in descending order of total downloads. + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 7 + }, + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 5, + }, + ] + ); + + // Add more extensions, including a new version of `ext1`, and backfilling + // an older version of `ext2`. + db.insert_extension_versions( + &[ + ( + "ext1", + vec![NewExtensionVersion { + name: "Extension One".into(), + version: semver::Version::parse("0.0.3").unwrap(), + description: "a real good extension".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + }], + ), + ( + "ext2", + vec![NewExtensionVersion { + name: "Extension Two".into(), + version: semver::Version::parse("0.1.0").unwrap(), + description: "an old extension".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + }], + ), + ] + .into_iter() + .collect(), + ) + .await + .unwrap(); + + let versions = db.get_known_extension_versions().await.unwrap(); + assert_eq!( + versions, + [ + ( + "ext1".into(), + vec!["0.0.1".into(), "0.0.2".into(), "0.0.3".into()] + ), + ("ext2".into(), vec!["0.1.0".into(), "0.2.0".into()]) + ] + .into_iter() + .collect() + ); + + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 7 + }, + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.3".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 5, + }, + ] + ); +} diff --git a/crates/collab/src/env.rs b/crates/collab/src/env.rs index 58c29b0205f598e720e8ba7f38b983d31082b5c2..4e6fe3b3a38b9402de779f7dd592cd1f9ed2ba0d 100644 --- a/crates/collab/src/env.rs +++ b/crates/collab/src/env.rs @@ -3,7 +3,8 @@ use std::fs; pub fn load_dotenv() -> anyhow::Result<()> { let env: toml::map::Map = toml::de::from_str( - &fs::read_to_string("./.env.toml").map_err(|_| anyhow!("no .env.toml file found"))?, + &fs::read_to_string("./crates/collab/.env.toml") + .map_err(|_| anyhow!("no .env.toml file found"))?, )?; for (key, value) in env { diff --git a/crates/collab/src/lib.rs b/crates/collab/src/lib.rs index aba9bd75d1f0aa9cc1849309dcb8f8db5b2ed9e3..195ed7b11d9b255cb444f4fb385361792eb137da 100644 --- a/crates/collab/src/lib.rs +++ b/crates/collab/src/lib.rs @@ -8,11 +8,14 @@ pub mod rpc; #[cfg(test)] mod tests; +use anyhow::anyhow; +use aws_config::{BehaviorVersion, Region}; use axum::{http::StatusCode, response::IntoResponse}; use db::Database; use executor::Executor; use serde::Deserialize; use std::{path::PathBuf, sync::Arc}; +use util::ResultExt; pub type Result = std::result::Result; @@ -100,6 +103,11 @@ pub struct Config { pub live_kit_secret: Option, pub rust_log: Option, pub log_json: Option, + pub blob_store_url: Option, + pub blob_store_region: Option, + pub blob_store_access_key: Option, + pub blob_store_secret_key: Option, + pub blob_store_bucket: Option, pub zed_environment: Arc, } @@ -118,6 +126,7 @@ pub struct MigrateConfig { pub struct AppState { pub db: Arc, pub live_kit_client: Option>, + pub blob_store_client: Option, pub config: Config, } @@ -146,8 +155,44 @@ impl AppState { let this = Self { db: Arc::new(db), live_kit_client, + blob_store_client: build_blob_store_client(&config).await.log_err(), config, }; Ok(Arc::new(this)) } } + +async fn build_blob_store_client(config: &Config) -> anyhow::Result { + let keys = aws_sdk_s3::config::Credentials::new( + config + .blob_store_access_key + .clone() + .ok_or_else(|| anyhow!("missing blob_store_access_key"))?, + config + .blob_store_secret_key + .clone() + .ok_or_else(|| anyhow!("missing blob_store_secret_key"))?, + None, + None, + "env", + ); + + let s3_config = aws_config::defaults(BehaviorVersion::latest()) + .endpoint_url( + config + .blob_store_url + .as_ref() + .ok_or_else(|| anyhow!("missing blob_store_url"))?, + ) + .region(Region::new( + config + .blob_store_region + .clone() + .ok_or_else(|| anyhow!("missing blob_store_region"))?, + )) + .credentials_provider(keys) + .load() + .await; + + Ok(aws_sdk_s3::Client::new(&s3_config)) +} diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index a2fda0dd333a7eda6a960dab7b36e6e8b5e96168..b80e8961df879838e329eea7dd30c54bea118711 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -1,6 +1,9 @@ use anyhow::anyhow; use axum::{routing::get, Extension, Router}; -use collab::{db, env, executor::Executor, AppState, Config, MigrateConfig, Result}; +use collab::{ + api::fetch_extensions_from_blob_store_periodically, db, env, executor::Executor, AppState, + Config, MigrateConfig, Result, +}; use db::Database; use std::{ env::args, @@ -50,6 +53,8 @@ async fn main() -> Result<()> { let rpc_server = collab::rpc::Server::new(epoch, state.clone(), Executor::Production); rpc_server.start().await?; + fetch_extensions_from_blob_store_periodically(state.clone(), Executor::Production); + let app = collab::api::routes(rpc_server.clone(), state.clone()) .merge(collab::rpc::routes(rpc_server.clone())) .merge( diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index 62870b860c2c17e20f9e9ccf8d6a5b4a4fc29b04..39292ead441fef92b32aab42fa9dbc4a331d2205 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -479,6 +479,7 @@ impl TestServer { Arc::new(AppState { db: test_db.db().clone(), live_kit_client: Some(Arc::new(fake_server.create_api_client())), + blob_store_client: None, config: Config { http_port: 0, database_url: "".into(), @@ -491,6 +492,11 @@ impl TestServer { rust_log: None, log_json: None, zed_environment: "test".into(), + blob_store_url: None, + blob_store_region: None, + blob_store_access_key: None, + blob_store_secret_key: None, + blob_store_bucket: None, }, }) } diff --git a/script/bootstrap b/script/bootstrap index 16ae872dbdd3c76bdb8cd5696f3d10670bf5dd71..054daccf4204eb1d41b66b578f563ea470fa9312 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,6 +3,10 @@ echo "installing foreman..." which foreman > /dev/null || brew install foreman +echo "installing minio..." +which minio > /dev/null || brew install minio/stable/minio +mkdir -p .blob_store/the-extensions-bucket + echo "creating database..." script/sqlx database create diff --git a/script/seed-db b/script/seed-db index 277ea89ba3b8e8dc056f6fab052531cec86cf102..5079e0195539536a071bc4bd05e5d7a0853eedd8 100755 --- a/script/seed-db +++ b/script/seed-db @@ -1,5 +1,4 @@ #!/bin/bash set -e -cd crates/collab cargo run --quiet --package=collab --features seed-support --bin seed -- $@