Max Brunsfeld , Marshall Bowers , Marshall , and Conrad created
This PR adds a REST API to the collab server for searching and
downloading extensions. Previously, we had implemented this API in
zed.dev directly, but this implementation is better, because we use the
collab database to store the download counts for extensions.
Release Notes:
- N/A
---------
Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>
Co-authored-by: Marshall <marshall@zed.dev>
Co-authored-by: Conrad <conrad@zed.dev>.github/workflows/ci.yml | 3
.github/workflows/deploy_collab.yml | 34
.gitignore | 5
Cargo.lock | 675 +++
Cargo.toml | 187
Procfile | 3
crates/collab/.env.toml | 5
crates/collab/Cargo.toml | 21
crates/collab/k8s/collab.template.yml | 25
crates/collab/migrations.sqlite/20221109000000_test_schema.sql | 22
crates/collab/migrations/20240214102900_add_extensions.sql | 22
crates/collab/src/api.rs | 5
crates/collab/src/api/extensions.rs | 237 +
crates/collab/src/db.rs | 55
crates/collab/src/db/ids.rs | 1
crates/collab/src/db/queries.rs | 1
crates/collab/src/db/queries/extensions.rs | 205 +
crates/collab/src/db/tables.rs | 2
crates/collab/src/db/tables/extension.rs | 27
crates/collab/src/db/tables/extension_version.rs | 36
crates/collab/src/db/tests.rs | 1
crates/collab/src/db/tests/extension_tests.rs | 219 +
crates/collab/src/env.rs | 3
crates/collab/src/lib.rs | 45
crates/collab/src/main.rs | 7
crates/collab/src/tests/test_server.rs | 6
script/bootstrap | 4
script/seed-db | 1
28 files changed, 1,691 insertions(+), 166 deletions(-)
@@ -35,6 +35,9 @@ jobs:
submodules: "recursive"
fetch-depth: 0
+ - name: Remove untracked files
+ run: git clean -df
+
- name: Set up default .cargo/config.toml
run: cp ./.cargo/ci-config.toml ~/.cargo/config.toml
@@ -45,8 +45,18 @@ jobs:
submodules: "recursive"
fetch-depth: 0
+ - name: Install cargo nextest
+ shell: bash -euxo pipefail {0}
+ run: |
+ cargo install cargo-nextest
+
+ - name: Limit target directory size
+ shell: bash -euxo pipefail {0}
+ run: script/clear-target-dir-if-larger-than 100
+
- name: Run tests
- uses: ./.github/actions/run_tests
+ shell: bash -euxo pipefail {0}
+ run: cargo nextest run --package collab --no-fail-fast
publish:
name: Publish collab server image
@@ -90,22 +100,26 @@ jobs:
- name: Sign into Kubernetes
run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
- - name: Determine namespace
+ - name: Start rollout
run: |
set -eu
if [[ $GITHUB_REF_NAME = "collab-production" ]]; then
- echo "Deploying collab:$GITHUB_SHA to production"
- echo "KUBE_NAMESPACE=production" >> $GITHUB_ENV
+ export ZED_KUBE_NAMESPACE=production
elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then
- echo "Deploying collab:$GITHUB_SHA to staging"
- echo "KUBE_NAMESPACE=staging" >> $GITHUB_ENV
+ export ZED_KUBE_NAMESPACE=staging
else
echo "cowardly refusing to deploy from an unknown branch"
exit 1
fi
- - name: Start rollout
- run: kubectl -n "$KUBE_NAMESPACE" set image deployment/collab collab=registry.digitalocean.com/zed/collab:${GITHUB_SHA}
+ echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"
+
+ source script/lib/deploy-helpers.sh
+ export_vars_for_environment $ZED_KUBE_NAMESPACE
+
+ export ZED_DO_CERTIFICATE_ID=$(doctl compute certificate list --format ID --no-header)
+ export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
- - name: Wait for rollout to finish
- run: kubectl -n "$KUBE_NAMESPACE" rollout status deployment/collab
+ envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
+ kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/collab --watch
+ echo "deployed collab.template.yml to ${ZED_KUBE_NAMESPACE}"
@@ -5,12 +5,8 @@
.DS_Store
/plugins/bin
/script/node_modules
-/styles/node_modules
-/styles/src/types/zed.ts
/crates/theme/schemas/theme.json
-/crates/collab/static/styles.css
/crates/collab/.admins.json
-/vendor/bin
/assets/*licenses.md
**/venv
.build
@@ -25,3 +21,4 @@ DerivedData/
**/*.db
.pytest_cache
.venv
+.blob_store
@@ -714,6 +714,368 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+[[package]]
+name = "aws-config"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7af266887e24cd5f6d2ea7433cacd25dcd4773b7f70e488701968a7cdf51df57"
+dependencies = [
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-sdk-sso",
+ "aws-sdk-ssooidc",
+ "aws-sdk-sts",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes 1.5.0",
+ "fastrand 2.0.0",
+ "hex",
+ "http 0.2.9",
+ "hyper",
+ "ring 0.17.7",
+ "time",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-credential-types"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d56f287a9e65e4914bfedb5b22c056b65e4c232fca512d5509a9df36386759f"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-runtime"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6a29eca8ea8982028a4df81883e7001e250a21d323b86418884b5345950a4b"
+dependencies = [
+ "aws-credential-types",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes 1.5.0",
+ "fastrand 2.0.0",
+ "http 0.2.9",
+ "http-body",
+ "percent-encoding",
+ "pin-project-lite 0.2.13",
+ "tracing",
+ "uuid 1.4.1",
+]
+
+[[package]]
+name = "aws-sdk-s3"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c977e92277652aefb9a76a0fca652b26757d6845dce0d7bf4426da80f13d85b0"
+dependencies = [
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-checksums",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "bytes 1.5.0",
+ "http 0.2.9",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "regex-lite",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "aws-sdk-sso"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d7f527c7b28af1a641f7d89f9e6a4863e8ec00f39d2b731b056fc5ec5ce829"
+dependencies = [
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes 1.5.0",
+ "http 0.2.9",
+ "once_cell",
+ "regex-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-ssooidc"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d0be3224cd574ee8ab5fd7c32087876f25c134c27ac603fcb38669ed8d346b0"
+dependencies = [
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes 1.5.0",
+ "http 0.2.9",
+ "once_cell",
+ "regex-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-sts"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b3167c60d82a13bbaef569da06041644ff41e85c6377e5dad53fa2526ccfe9d"
+dependencies = [
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-query",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "http 0.2.9",
+ "once_cell",
+ "regex-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sigv4"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54b1cbe0eee57a213039088dbdeca7be9352f24e0d72332d961e8a1cb388f82d"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes 1.5.0",
+ "crypto-bigint 0.5.5",
+ "form_urlencoded",
+ "hex",
+ "hmac 0.12.1",
+ "http 0.2.9",
+ "http 1.0.0",
+ "once_cell",
+ "p256",
+ "percent-encoding",
+ "ring 0.17.7",
+ "sha2 0.10.7",
+ "subtle",
+ "time",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-async"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "426a5bc369ca7c8d3686439e46edc727f397a47ab3696b13f3ae8c81b3b36132"
+dependencies = [
+ "futures-util",
+ "pin-project-lite 0.2.13",
+ "tokio",
+]
+
+[[package]]
+name = "aws-smithy-checksums"
+version = "0.60.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ee554133eca2611b66d23548e48f9b44713befdb025ab76bc00185b878397a1"
+dependencies = [
+ "aws-smithy-http",
+ "aws-smithy-types",
+ "bytes 1.5.0",
+ "crc32c",
+ "crc32fast",
+ "hex",
+ "http 0.2.9",
+ "http-body",
+ "md-5",
+ "pin-project-lite 0.2.13",
+ "sha1",
+ "sha2 0.10.7",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-eventstream"
+version = "0.60.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858"
+dependencies = [
+ "aws-smithy-types",
+ "bytes 1.5.0",
+ "crc32fast",
+]
+
+[[package]]
+name = "aws-smithy-http"
+version = "0.60.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85d6a0619f7b67183067fa3b558f94f90753da2df8c04aeb7336d673f804b0b8"
+dependencies = [
+ "aws-smithy-eventstream",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes 1.5.0",
+ "bytes-utils",
+ "futures-core",
+ "http 0.2.9",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite 0.2.13",
+ "pin-utils",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-json"
+version = "0.60.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1c1b5186b6f5c579bf0de1bcca9dd3d946d6d51361ea1d18131f6a0b64e13ae"
+dependencies = [
+ "aws-smithy-types",
+]
+
+[[package]]
+name = "aws-smithy-query"
+version = "0.60.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c0a2ce65882e788d2cf83ff28b9b16918de0460c47bf66c5da4f6c17b4c9694"
+dependencies = [
+ "aws-smithy-types",
+ "urlencoding",
+]
+
+[[package]]
+name = "aws-smithy-runtime"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4cb6b3afa5fc9825a75675975dcc3e21764b5476bc91dbc63df4ea3d30a576e"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes 1.5.0",
+ "fastrand 2.0.0",
+ "h2",
+ "http 0.2.9",
+ "http-body",
+ "hyper",
+ "hyper-rustls",
+ "once_cell",
+ "pin-project-lite 0.2.13",
+ "pin-utils",
+ "rustls",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-runtime-api"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23165433e80c04e8c09cee66d171292ae7234bae05fa9d5636e33095eae416b2"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-types",
+ "bytes 1.5.0",
+ "http 0.2.9",
+ "pin-project-lite 0.2.13",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-types"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c94a5bec34850b92c9a054dad57b95c1d47f25125f55973e19f6ad788f0381ff"
+dependencies = [
+ "base64-simd",
+ "bytes 1.5.0",
+ "bytes-utils",
+ "futures-core",
+ "http 0.2.9",
+ "http-body",
+ "itoa",
+ "num-integer",
+ "pin-project-lite 0.2.13",
+ "pin-utils",
+ "ryu",
+ "serde",
+ "time",
+ "tokio",
+ "tokio-util 0.7.9",
+]
+
+[[package]]
+name = "aws-smithy-xml"
+version = "0.60.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d16f94c9673412b7a72e3c3efec8de89081c320bf59ea12eed34c417a62ad600"
+dependencies = [
+ "xmlparser",
+]
+
+[[package]]
+name = "aws-types"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ff7e122ee50ca962e9de91f5850cc37e2184b1219611eef6d44aa85929b54f6"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "http 0.2.9",
+ "rustc_version",
+ "tracing",
+]
+
[[package]]
name = "axum"
version = "0.5.17"
@@ -727,7 +1089,7 @@ dependencies = [
"bytes 1.5.0",
"futures-util",
"headers",
- "http",
+ "http 0.2.9",
"http-body",
"hyper",
"itoa",
@@ -758,7 +1120,7 @@ dependencies = [
"async-trait",
"bytes 1.5.0",
"futures-util",
- "http",
+ "http 0.2.9",
"http-body",
"mime",
"tower-layer",
@@ -774,7 +1136,7 @@ dependencies = [
"axum",
"bytes 1.5.0",
"futures-util",
- "http",
+ "http 0.2.9",
"mime",
"pin-project-lite 0.2.13",
"serde",
@@ -812,6 +1174,12 @@ dependencies = [
"nix 0.23.2",
]
+[[package]]
+name = "base16ct"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
+
[[package]]
name = "base64"
version = "0.13.1"
@@ -824,6 +1192,16 @@ version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
+[[package]]
+name = "base64-simd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
+dependencies = [
+ "outref",
+ "vsimd",
+]
+
[[package]]
name = "base64ct"
version = "1.6.0"
@@ -1169,6 +1547,16 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+[[package]]
+name = "bytes-utils"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
+dependencies = [
+ "bytes 1.5.0",
+ "either",
+]
+
[[package]]
name = "call"
version = "0.1.0"
@@ -1537,6 +1925,8 @@ dependencies = [
"async-trait",
"async-tungstenite",
"audio",
+ "aws-config",
+ "aws-sdk-s3",
"axum",
"axum-extra",
"base64 0.13.1",
@@ -1582,6 +1972,7 @@ dependencies = [
"rpc",
"scrypt",
"sea-orm",
+ "semver",
"serde",
"serde_derive",
"serde_json",
@@ -2087,6 +2478,15 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
+[[package]]
+name = "crc32c"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2"
+dependencies = [
+ "rustc_version",
+]
+
[[package]]
name = "crc32fast"
version = "1.3.2"
@@ -2149,6 +2549,28 @@ dependencies = [
"cfg-if 1.0.0",
]
+[[package]]
+name = "crypto-bigint"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
+dependencies = [
+ "generic-array",
+ "rand_core 0.6.4",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "crypto-bigint"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "crypto-common"
version = "0.1.6"
@@ -2161,9 +2583,9 @@ dependencies = [
[[package]]
name = "crypto-mac"
-version = "0.11.1"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
+checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e"
dependencies = [
"generic-array",
"subtle",
@@ -2286,6 +2708,16 @@ dependencies = [
"byteorder",
]
+[[package]]
+name = "der"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
+dependencies = [
+ "const-oid",
+ "zeroize",
+]
+
[[package]]
name = "der"
version = "0.7.8"
@@ -2513,6 +2945,18 @@ version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd"
+[[package]]
+name = "ecdsa"
+version = "0.14.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
+dependencies = [
+ "der 0.6.1",
+ "elliptic-curve",
+ "rfc6979",
+ "signature 1.6.4",
+]
+
[[package]]
name = "editor"
version = "0.1.0"
@@ -2579,6 +3023,26 @@ dependencies = [
"serde",
]
+[[package]]
+name = "elliptic-curve"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
+dependencies = [
+ "base16ct",
+ "crypto-bigint 0.4.9",
+ "der 0.6.1",
+ "digest 0.10.7",
+ "ff",
+ "generic-array",
+ "group",
+ "pkcs8 0.9.0",
+ "rand_core 0.6.4",
+ "sec1",
+ "subtle",
+ "zeroize",
+]
+
[[package]]
name = "encode_unicode"
version = "0.3.6"
@@ -2821,6 +3285,16 @@ dependencies = [
"workspace",
]
+[[package]]
+name = "ff"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "file_finder"
version = "0.1.0"
@@ -3160,9 +3634,9 @@ dependencies = [
[[package]]
name = "futures-channel"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
+checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
dependencies = [
"futures-core",
"futures-sink",
@@ -3170,9 +3644,9 @@ dependencies = [
[[package]]
name = "futures-core"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
[[package]]
name = "futures-executor"
@@ -3198,9 +3672,9 @@ dependencies = [
[[package]]
name = "futures-io"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-lite"
@@ -3219,9 +3693,9 @@ dependencies = [
[[package]]
name = "futures-macro"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
@@ -3230,21 +3704,21 @@ dependencies = [
[[package]]
name = "futures-sink"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
+checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
[[package]]
name = "futures-task"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
[[package]]
name = "futures-util"
-version = "0.3.28"
+version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
dependencies = [
"futures 0.1.31",
"futures-channel",
@@ -3546,6 +4020,17 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df00eed8d1f0db937f6be10e46e8072b0671accb504cf0f959c5c52c679f5b9"
+[[package]]
+name = "group"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
+dependencies = [
+ "ff",
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "h2"
version = "0.3.21"
@@ -3557,7 +4042,7 @@ dependencies = [
"futures-core",
"futures-sink",
"futures-util",
- "http",
+ "http 0.2.9",
"indexmap 1.9.3",
"slab",
"tokio",
@@ -3611,7 +4096,7 @@ dependencies = [
"base64 0.21.4",
"bytes 1.5.0",
"headers-core",
- "http",
+ "http 0.2.9",
"httpdate",
"mime",
"sha1",
@@ -3623,7 +4108,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
dependencies = [
- "http",
+ "http 0.2.9",
]
[[package]]
@@ -3736,6 +4221,17 @@ dependencies = [
"itoa",
]
+[[package]]
+name = "http"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea"
+dependencies = [
+ "bytes 1.5.0",
+ "fnv",
+ "itoa",
+]
+
[[package]]
name = "http-body"
version = "0.4.5"
@@ -3743,7 +4239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
dependencies = [
"bytes 1.5.0",
- "http",
+ "http 0.2.9",
"pin-project-lite 0.2.13",
]
@@ -3788,7 +4284,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2",
- "http",
+ "http 0.2.9",
"http-body",
"httparse",
"httpdate",
@@ -3801,6 +4297,22 @@ dependencies = [
"want",
]
+[[package]]
+name = "hyper-rustls"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
+dependencies = [
+ "futures-util",
+ "http 0.2.9",
+ "hyper",
+ "log",
+ "rustls",
+ "rustls-native-certs",
+ "tokio",
+ "tokio-rustls",
+]
+
[[package]]
name = "hyper-timeout"
version = "0.4.1"
@@ -4033,7 +4545,7 @@ dependencies = [
"encoding_rs",
"event-listener",
"futures-lite",
- "http",
+ "http 0.2.9",
"log",
"mime",
"once_cell",
@@ -5578,12 +6090,29 @@ dependencies = [
"workspace",
]
+[[package]]
+name = "outref"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
+
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+[[package]]
+name = "p256"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
+dependencies = [
+ "ecdsa",
+ "elliptic-curve",
+ "sha2 0.10.7",
+]
+
[[package]]
name = "palette"
version = "0.7.3"
@@ -5688,9 +6217,9 @@ dependencies = [
[[package]]
name = "password-hash"
-version = "0.2.3"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7"
+checksum = "c1a5d4e9c205d2c1ae73b84aab6240e98218c0e72e63b50422cfb2d1ca952282"
dependencies = [
"base64ct",
"rand_core 0.6.4",
@@ -5902,9 +6431,19 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
dependencies = [
- "der",
- "pkcs8",
- "spki",
+ "der 0.7.8",
+ "pkcs8 0.10.2",
+ "spki 0.7.2",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba"
+dependencies = [
+ "der 0.6.1",
+ "spki 0.6.0",
]
[[package]]
@@ -5913,8 +6452,8 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
- "der",
- "spki",
+ "der 0.7.8",
+ "spki 0.7.2",
]
[[package]]
@@ -6705,6 +7244,12 @@ dependencies = [
"regex-syntax 0.8.2",
]
+[[package]]
+name = "regex-lite"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e"
+
[[package]]
name = "regex-syntax"
version = "0.6.29"
@@ -6752,7 +7297,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2",
- "http",
+ "http 0.2.9",
"http-body",
"hyper",
"hyper-tls",
@@ -6793,6 +7338,17 @@ dependencies = [
"usvg",
]
+[[package]]
+name = "rfc6979"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb"
+dependencies = [
+ "crypto-bigint 0.4.9",
+ "hmac 0.12.1",
+ "zeroize",
+]
+
[[package]]
name = "rgb"
version = "0.8.36"
@@ -6831,11 +7387,25 @@ dependencies = [
"libc",
"once_cell",
"spin 0.5.2",
- "untrusted",
+ "untrusted 0.7.1",
"web-sys",
"winapi 0.3.9",
]
+[[package]]
+name = "ring"
+version = "0.17.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74"
+dependencies = [
+ "cc",
+ "getrandom 0.2.10",
+ "libc",
+ "spin 0.9.8",
+ "untrusted 0.9.0",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "rkyv"
version = "0.7.42"
@@ -6989,10 +7559,10 @@ dependencies = [
"num-iter",
"num-traits",
"pkcs1",
- "pkcs8",
+ "pkcs8 0.10.2",
"rand_core 0.6.4",
- "signature",
- "spki",
+ "signature 2.1.0",
+ "spki 0.7.2",
"subtle",
"zeroize",
]
@@ -7124,15 +7694,28 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.21.7"
+version = "0.21.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
+checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba"
dependencies = [
- "ring",
+ "log",
+ "ring 0.17.7",
"rustls-webpki",
"sct",
]
+[[package]]
+name = "rustls-native-certs"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
+dependencies = [
+ "openssl-probe",
+ "rustls-pemfile",
+ "schannel",
+ "security-framework",
+]
+
[[package]]
name = "rustls-pemfile"
version = "1.0.3"
@@ -7144,12 +7727,12 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.101.6"
+version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
- "ring",
- "untrusted",
+ "ring 0.17.7",
+ "untrusted 0.9.0",
]
[[package]]
@@ -7302,8 +7885,8 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
dependencies = [
- "ring",
- "untrusted",
+ "ring 0.16.20",
+ "untrusted 0.7.1",
]
[[package]]
@@ -1,91 +1,91 @@
[workspace]
members = [
- "crates/activity_indicator",
- "crates/ai",
- "crates/assets",
- "crates/assistant",
- "crates/audio",
- "crates/auto_update",
- "crates/breadcrumbs",
- "crates/call",
- "crates/channel",
- "crates/cli",
- "crates/client",
- "crates/clock",
- "crates/collab",
- "crates/collab_ui",
- "crates/collections",
- "crates/command_palette",
- "crates/copilot",
- "crates/copilot_ui",
- "crates/db",
- "crates/diagnostics",
- "crates/editor",
- "crates/extension",
- "crates/extensions_ui",
- "crates/feature_flags",
- "crates/feedback",
- "crates/file_finder",
- "crates/fs",
- "crates/fsevent",
- "crates/fuzzy",
- "crates/git",
- "crates/go_to_line",
- "crates/gpui",
- "crates/gpui_macros",
- "crates/install_cli",
- "crates/journal",
- "crates/language",
- "crates/language_selector",
- "crates/language_tools",
- "crates/live_kit_client",
- "crates/live_kit_server",
- "crates/lsp",
- "crates/markdown_preview",
- "crates/media",
- "crates/menu",
- "crates/multi_buffer",
- "crates/node_runtime",
- "crates/notifications",
- "crates/outline",
- "crates/picker",
- "crates/plugin",
- "crates/plugin_macros",
- "crates/prettier",
- "crates/project",
- "crates/project_panel",
- "crates/project_symbols",
- "crates/quick_action_bar",
- "crates/recent_projects",
- "crates/refineable",
- "crates/refineable/derive_refineable",
- "crates/release_channel",
- "crates/rich_text",
- "crates/rope",
- "crates/rpc",
- "crates/search",
- "crates/semantic_index",
- "crates/settings",
- "crates/snippet",
- "crates/sqlez",
- "crates/sqlez_macros",
- "crates/story",
- "crates/storybook",
- "crates/sum_tree",
- "crates/terminal",
- "crates/terminal_view",
- "crates/text",
- "crates/theme",
- "crates/theme_importer",
- "crates/theme_selector",
- "crates/ui",
- "crates/util",
- "crates/vcs_menu",
- "crates/vim",
- "crates/welcome",
- "crates/workspace",
- "crates/zed",
- "crates/zed_actions",
+ "crates/activity_indicator",
+ "crates/ai",
+ "crates/assets",
+ "crates/assistant",
+ "crates/audio",
+ "crates/auto_update",
+ "crates/breadcrumbs",
+ "crates/call",
+ "crates/channel",
+ "crates/cli",
+ "crates/client",
+ "crates/clock",
+ "crates/collab",
+ "crates/collab_ui",
+ "crates/collections",
+ "crates/command_palette",
+ "crates/copilot",
+ "crates/copilot_ui",
+ "crates/db",
+ "crates/diagnostics",
+ "crates/editor",
+ "crates/extension",
+ "crates/extensions_ui",
+ "crates/feature_flags",
+ "crates/feedback",
+ "crates/file_finder",
+ "crates/fs",
+ "crates/fsevent",
+ "crates/fuzzy",
+ "crates/git",
+ "crates/go_to_line",
+ "crates/gpui",
+ "crates/gpui_macros",
+ "crates/install_cli",
+ "crates/journal",
+ "crates/language",
+ "crates/language_selector",
+ "crates/language_tools",
+ "crates/live_kit_client",
+ "crates/live_kit_server",
+ "crates/lsp",
+ "crates/markdown_preview",
+ "crates/media",
+ "crates/menu",
+ "crates/multi_buffer",
+ "crates/node_runtime",
+ "crates/notifications",
+ "crates/outline",
+ "crates/picker",
+ "crates/plugin",
+ "crates/plugin_macros",
+ "crates/prettier",
+ "crates/project",
+ "crates/project_panel",
+ "crates/project_symbols",
+ "crates/quick_action_bar",
+ "crates/recent_projects",
+ "crates/refineable",
+ "crates/refineable/derive_refineable",
+ "crates/release_channel",
+ "crates/rich_text",
+ "crates/rope",
+ "crates/rpc",
+ "crates/search",
+ "crates/semantic_index",
+ "crates/settings",
+ "crates/snippet",
+ "crates/sqlez",
+ "crates/sqlez_macros",
+ "crates/story",
+ "crates/storybook",
+ "crates/sum_tree",
+ "crates/terminal",
+ "crates/terminal_view",
+ "crates/text",
+ "crates/theme",
+ "crates/theme_importer",
+ "crates/theme_selector",
+ "crates/ui",
+ "crates/util",
+ "crates/vcs_menu",
+ "crates/vim",
+ "crates/welcome",
+ "crates/workspace",
+ "crates/zed",
+ "crates/zed_actions",
]
default-members = ["crates/zed"]
resolver = "2"
@@ -191,8 +191,8 @@ globset = "0.4"
indoc = "1"
# We explicitly disable a http2 support in isahc.
isahc = { version = "1.7.2", default-features = false, features = [
- "static-curl",
- "text-decoding",
+ "static-curl",
+ "text-decoding",
] }
lazy_static = "1.4.0"
log = { version = "0.4.16", features = ["kv_unstable_serde"] }
@@ -208,12 +208,13 @@ regex = "1.5"
rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] }
rust-embed = { version = "8.0", features = ["include-exclude"] }
schemars = "0.8"
+semver = { version = "1.0" }
serde = { version = "1.0", features = ["derive", "rc"] }
serde_derive = { version = "1.0", features = ["deserialize_in_place"] }
serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] }
serde_json_lenient = { version = "0.1", features = [
- "preserve_order",
- "raw_value",
+ "preserve_order",
+ "raw_value",
] }
serde_repr = "0.1"
smallvec = { version = "1.6", features = ["union"] }
@@ -223,7 +224,11 @@ sysinfo = "0.29.10"
tempfile = "3.9.0"
thiserror = "1.0.29"
tiktoken-rs = "0.5.7"
-time = { version = "0.3", features = ["serde", "serde-well-known"] }
+time = { version = "0.3", features = [
+ "serde",
+ "serde-well-known",
+ "formatting",
+] }
toml = "0.5"
tree-sitter = { version = "0.20", features = ["wasm"] }
tree-sitter-astro = { git = "https://github.com/virchau13/tree-sitter-astro.git", rev = "e924787e12e8a03194f36a113290ac11d6dc10f3" }
@@ -1,2 +1,3 @@
-collab: cd crates/collab && RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run serve
+collab: RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run --package=collab serve
livekit: livekit-server --dev
+blob_store: MINIO_ROOT_USER=the-blob-store-access-key MINIO_ROOT_PASSWORD=the-blob-store-secret-key minio server .blob_store
@@ -7,6 +7,11 @@ ZED_ENVIRONMENT = "development"
LIVE_KIT_SERVER = "http://localhost:7880"
LIVE_KIT_KEY = "devkey"
LIVE_KIT_SECRET = "secret"
+BLOB_STORE_ACCESS_KEY = "the-blob-store-access-key"
+BLOB_STORE_SECRET_KEY = "the-blob-store-secret-key"
+BLOB_STORE_BUCKET = "the-extensions-bucket"
+BLOB_STORE_URL = "http://127.0.0.1:9000"
+BLOB_STORE_REGION = "the-region"
# RUST_LOG=info
# LOG_JSON=true
@@ -15,9 +15,11 @@ name = "seed"
required-features = ["seed-support"]
[dependencies]
+axum = { version = "0.5", features = ["json", "headers", "ws"] }
anyhow.workspace = true
+aws-config = { version = "1.1.5" }
+aws-sdk-s3 = { version = "1.15.0" }
async-tungstenite = "0.16"
-axum = { version = "0.5", features = ["json", "headers", "ws"] }
axum-extra = { version = "0.3", features = ["erased-json"] }
base64 = "0.13"
chrono.workspace = true
@@ -40,13 +42,26 @@ rand.workspace = true
reqwest = { version = "0.11", features = ["json"], optional = true }
rpc.workspace = true
scrypt = "0.7"
-sea-orm = { version = "0.12.x", features = ["sqlx-postgres", "postgres-array", "runtime-tokio-rustls", "with-uuid"] }
+sea-orm = { version = "0.12.x", features = [
+ "sqlx-postgres",
+ "postgres-array",
+ "runtime-tokio-rustls",
+ "with-uuid",
+] }
+semver.workspace = true
serde.workspace = true
serde_derive.workspace = true
serde_json.workspace = true
sha-1 = "0.9"
smallvec.workspace = true
-sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "any"] }
+sqlx = { version = "0.7", features = [
+ "runtime-tokio-rustls",
+ "postgres",
+ "json",
+ "time",
+ "uuid",
+ "any",
+] }
text.workspace = true
time.workspace = true
tokio = { version = "1", features = ["full"] }
@@ -105,6 +105,31 @@ spec:
secretKeyRef:
name: livekit
key: secret
+ - name: BLOB_STORE_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ name: blob-store
+ key: access_key
+ - name: BLOB_STORE_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ name: blob-store
+ key: secret_key
+ - name: BLOB_STORE_URL
+ valueFrom:
+ secretKeyRef:
+ name: blob-store
+ key: url
+ - name: BLOB_STORE_REGION
+ valueFrom:
+ secretKeyRef:
+ name: blob-store
+ key: region
+ - name: BLOB_STORE_BUCKET
+ valueFrom:
+ secretKeyRef:
+ name: blob-store
+ key: bucket
- name: INVITE_LINK_PREFIX
value: ${INVITE_LINK_PREFIX}
- name: RUST_BACKTRACE
@@ -353,3 +353,25 @@ CREATE TABLE contributors (
signed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (user_id)
);
+
+CREATE TABLE extensions (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ external_id TEXT NOT NULL,
+ name TEXT NOT NULL,
+ latest_version TEXT NOT NULL,
+ total_download_count INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE extension_versions (
+ extension_id INTEGER REFERENCES extensions(id),
+ version TEXT NOT NULL,
+ published_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ authors TEXT NOT NULL,
+ repository TEXT NOT NULL,
+ description TEXT NOT NULL,
+ download_count INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (extension_id, version)
+);
+
+CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id");
+CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count");
@@ -0,0 +1,22 @@
+CREATE TABLE IF NOT EXISTS extensions (
+ id SERIAL PRIMARY KEY,
+ name TEXT NOT NULL,
+ external_id TEXT NOT NULL,
+ latest_version TEXT NOT NULL,
+ total_download_count BIGINT NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS extension_versions (
+ extension_id INTEGER REFERENCES extensions(id),
+ version TEXT NOT NULL,
+ published_at TIMESTAMP NOT NULL DEFAULT now(),
+ authors TEXT NOT NULL,
+ repository TEXT NOT NULL,
+ description TEXT NOT NULL,
+ download_count BIGINT NOT NULL DEFAULT 0,
+ PRIMARY KEY(extension_id, version)
+);
+
+CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id");
+CREATE INDEX "trigram_index_extensions_name" ON "extensions" USING GIN(name gin_trgm_ops);
+CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count");
@@ -1,3 +1,5 @@
+mod extensions;
+
use crate::{
auth,
db::{ContributorSelector, User, UserId},
@@ -20,6 +22,8 @@ use std::sync::Arc;
use tower::ServiceBuilder;
use tracing::instrument;
+pub use extensions::fetch_extensions_from_blob_store_periodically;
+
pub fn routes(rpc_server: Arc<rpc::Server>, state: Arc<AppState>) -> Router<Body> {
Router::new()
.route("/user", get(get_authenticated_user))
@@ -28,6 +32,7 @@ pub fn routes(rpc_server: Arc<rpc::Server>, state: Arc<AppState>) -> Router<Body
.route("/rpc_server_snapshot", get(get_rpc_server_snapshot))
.route("/contributors", get(get_contributors).post(add_contributor))
.route("/contributor", get(check_is_contributor))
+ .merge(extensions::router())
.layer(
ServiceBuilder::new()
.layer(Extension(state))
@@ -0,0 +1,237 @@
+use crate::{
+ db::{ExtensionMetadata, NewExtensionVersion},
+ executor::Executor,
+ AppState, Error, Result,
+};
+use anyhow::{anyhow, Context as _};
+use aws_sdk_s3::presigning::PresigningConfig;
+use axum::{
+ extract::{Path, Query},
+ response::Redirect,
+ routing::get,
+ Extension, Json, Router,
+};
+use collections::HashMap;
+use hyper::StatusCode;
+use serde::{Deserialize, Serialize};
+use std::{sync::Arc, time::Duration};
+use time::PrimitiveDateTime;
+use util::ResultExt;
+
+pub fn router() -> Router {
+ Router::new()
+ .route("/extensions", get(get_extensions))
+ .route(
+ "/extensions/:extension_id/:version/download",
+ get(download_extension),
+ )
+}
+
+#[derive(Debug, Deserialize)]
+struct GetExtensionsParams {
+ filter: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct DownloadExtensionParams {
+ extension_id: String,
+ version: String,
+}
+
+#[derive(Debug, Serialize)]
+struct GetExtensionsResponse {
+ pub data: Vec<ExtensionMetadata>,
+}
+
+#[derive(Deserialize)]
+struct ExtensionManifest {
+ name: String,
+ version: String,
+ description: Option<String>,
+ authors: Vec<String>,
+ repository: String,
+}
+
+async fn get_extensions(
+ Extension(app): Extension<Arc<AppState>>,
+ Query(params): Query<GetExtensionsParams>,
+) -> Result<Json<GetExtensionsResponse>> {
+ let extensions = app.db.get_extensions(params.filter.as_deref(), 30).await?;
+ Ok(Json(GetExtensionsResponse { data: extensions }))
+}
+
+async fn download_extension(
+ Extension(app): Extension<Arc<AppState>>,
+ Path(params): Path<DownloadExtensionParams>,
+) -> Result<Redirect> {
+ let Some((blob_store_client, bucket)) = app
+ .blob_store_client
+ .clone()
+ .zip(app.config.blob_store_bucket.clone())
+ else {
+ Err(Error::Http(
+ StatusCode::NOT_IMPLEMENTED,
+ "not supported".into(),
+ ))?
+ };
+
+ let DownloadExtensionParams {
+ extension_id,
+ version,
+ } = params;
+
+ let version_exists = app
+ .db
+ .record_extension_download(&extension_id, &version)
+ .await?;
+
+ if !version_exists {
+ Err(Error::Http(
+ StatusCode::NOT_FOUND,
+ "unknown extension version".into(),
+ ))?;
+ }
+
+ let url = blob_store_client
+ .get_object()
+ .bucket(bucket)
+ .key(format!(
+ "extensions/{extension_id}/{version}/archive.tar.gz"
+ ))
+ .presigned(PresigningConfig::expires_in(EXTENSION_DOWNLOAD_URL_LIFETIME).unwrap())
+ .await
+ .map_err(|e| anyhow!("failed to create presigned extension download url {e}"))?;
+
+ Ok(Redirect::temporary(url.uri()))
+}
+
+const EXTENSION_FETCH_INTERVAL: Duration = Duration::from_secs(5 * 60);
+const EXTENSION_DOWNLOAD_URL_LIFETIME: Duration = Duration::from_secs(3 * 60);
+
+pub fn fetch_extensions_from_blob_store_periodically(app_state: Arc<AppState>, executor: Executor) {
+ let Some(blob_store_client) = app_state.blob_store_client.clone() else {
+ log::info!("no blob store client");
+ return;
+ };
+ let Some(blob_store_bucket) = app_state.config.blob_store_bucket.clone() else {
+ log::info!("no blob store bucket");
+ return;
+ };
+
+ executor.spawn_detached({
+ let executor = executor.clone();
+ async move {
+ loop {
+ fetch_extensions_from_blob_store(
+ &blob_store_client,
+ &blob_store_bucket,
+ &app_state,
+ )
+ .await
+ .log_err();
+ executor.sleep(EXTENSION_FETCH_INTERVAL).await;
+ }
+ }
+ });
+}
+
+async fn fetch_extensions_from_blob_store(
+ blob_store_client: &aws_sdk_s3::Client,
+ blob_store_bucket: &String,
+ app_state: &Arc<AppState>,
+) -> anyhow::Result<()> {
+ let list = blob_store_client
+ .list_objects()
+ .bucket(blob_store_bucket)
+ .prefix("extensions/")
+ .send()
+ .await?;
+
+ let objects = list
+ .contents
+ .ok_or_else(|| anyhow!("missing bucket contents"))?;
+
+ let mut published_versions = HashMap::<&str, Vec<&str>>::default();
+ for object in &objects {
+ let Some(key) = object.key.as_ref() else {
+ continue;
+ };
+ let mut parts = key.split('/');
+ let Some(_) = parts.next().filter(|part| *part == "extensions") else {
+ continue;
+ };
+ let Some(extension_id) = parts.next() else {
+ continue;
+ };
+ let Some(version) = parts.next() else {
+ continue;
+ };
+ published_versions
+ .entry(extension_id)
+ .or_default()
+ .push(version);
+ }
+
+ let known_versions = app_state.db.get_known_extension_versions().await?;
+
+ let mut new_versions = HashMap::<&str, Vec<NewExtensionVersion>>::default();
+ let empty = Vec::new();
+ for (extension_id, published_versions) in published_versions {
+ let known_versions = known_versions.get(extension_id).unwrap_or(&empty);
+
+ for published_version in published_versions {
+ if known_versions
+ .binary_search_by_key(&published_version, String::as_str)
+ .is_err()
+ {
+ let object = blob_store_client
+ .get_object()
+ .bucket(blob_store_bucket)
+ .key(format!(
+ "extensions/{extension_id}/{published_version}/manifest.json"
+ ))
+ .send()
+ .await?;
+ let manifest_bytes = object
+ .body
+ .collect()
+ .await
+ .map(|data| data.into_bytes())
+ .with_context(|| format!("failed to download manifest for extension {extension_id} version {published_version}"))?
+ .to_vec();
+ let manifest = serde_json::from_slice::<ExtensionManifest>(&manifest_bytes)
+ .with_context(|| format!("invalid manifest for extension {extension_id} version {published_version}: {}", String::from_utf8_lossy(&manifest_bytes)))?;
+
+ let published_at = object.last_modified.ok_or_else(|| anyhow!("missing last modified timestamp for extension {extension_id} version {published_version}"))?;
+ let published_at =
+ time::OffsetDateTime::from_unix_timestamp_nanos(published_at.as_nanos())?;
+ let published_at = PrimitiveDateTime::new(published_at.date(), published_at.time());
+
+ let version = semver::Version::parse(&manifest.version).with_context(|| {
+ format!(
+ "invalid version for extension {extension_id} version {published_version}"
+ )
+ })?;
+
+ new_versions
+ .entry(extension_id)
+ .or_default()
+ .push(NewExtensionVersion {
+ name: manifest.name,
+ version,
+ description: manifest.description.unwrap_or_default(),
+ authors: manifest.authors,
+ repository: manifest.repository,
+ published_at,
+ });
+ }
+ }
+ }
+
+ app_state
+ .db
+ .insert_extension_versions(&new_versions)
+ .await?;
+
+ Ok(())
+}
@@ -1,12 +1,8 @@
-#[cfg(test)]
-pub mod tests;
-
-#[cfg(test)]
-pub use tests::TestDb;
-
mod ids;
mod queries;
mod tables;
+#[cfg(test)]
+pub mod tests;
use crate::{executor::Executor, Error, Result};
use anyhow::anyhow;
@@ -25,7 +21,7 @@ use sea_orm::{
FromQueryResult, IntoActiveModel, IsolationLevel, JoinType, QueryOrder, QuerySelect, Statement,
TransactionTrait,
};
-use serde::{Deserialize, Serialize};
+use serde::{ser::Error as _, Deserialize, Serialize, Serializer};
use sqlx::{
migrate::{Migrate, Migration, MigrationSource},
Connection,
@@ -40,13 +36,17 @@ use std::{
sync::Arc,
time::Duration,
};
-pub use tables::*;
+use time::{format_description::well_known::iso8601, PrimitiveDateTime};
use tokio::sync::{Mutex, OwnedMutexGuard};
+#[cfg(test)]
+pub use tests::TestDb;
+
pub use ids::*;
pub use queries::contributors::ContributorSelector;
pub use sea_orm::ConnectOptions;
pub use tables::user::Model as User;
+pub use tables::*;
/// Database gives you a handle that lets you access the database.
/// It handles pooling internally.
@@ -717,3 +717,42 @@ pub struct WorktreeSettingsFile {
pub path: String,
pub content: String,
}
+
+pub struct NewExtensionVersion {
+ pub name: String,
+ pub version: semver::Version,
+ pub description: String,
+ pub authors: Vec<String>,
+ pub repository: String,
+ pub published_at: PrimitiveDateTime,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct ExtensionMetadata {
+ pub id: String,
+ pub name: String,
+ pub version: String,
+ pub authors: Vec<String>,
+ pub repository: String,
+ #[serde(serialize_with = "serialize_iso8601")]
+ pub published_at: PrimitiveDateTime,
+ pub download_count: u64,
+}
+
+pub fn serialize_iso8601<S: Serializer>(
+ datetime: &PrimitiveDateTime,
+ serializer: S,
+) -> Result<S::Ok, S::Error> {
+ const SERDE_CONFIG: iso8601::EncodedConfig = iso8601::Config::DEFAULT
+ .set_year_is_six_digits(false)
+ .set_time_precision(iso8601::TimePrecision::Second {
+ decimal_digits: None,
+ })
+ .encode();
+
+ datetime
+ .assume_utc()
+ .format(&time::format_description::well_known::Iso8601::<SERDE_CONFIG>)
+ .map_err(S::Error::custom)?
+ .serialize(serializer)
+}
@@ -85,6 +85,7 @@ id_type!(SignupId);
id_type!(UserId);
id_type!(ChannelBufferCollaboratorId);
id_type!(FlagId);
+id_type!(ExtensionId);
id_type!(NotificationId);
id_type!(NotificationKindId);
@@ -5,6 +5,7 @@ pub mod buffers;
pub mod channels;
pub mod contacts;
pub mod contributors;
+pub mod extensions;
pub mod messages;
pub mod notifications;
pub mod projects;
@@ -0,0 +1,205 @@
+use super::*;
+
+impl Database {
+ pub async fn get_extensions(
+ &self,
+ filter: Option<&str>,
+ limit: usize,
+ ) -> Result<Vec<ExtensionMetadata>> {
+ self.transaction(|tx| async move {
+ let mut condition = Condition::all();
+ if let Some(filter) = filter {
+ let fuzzy_name_filter = Self::fuzzy_like_string(filter);
+ condition = condition.add(Expr::cust_with_expr("name ILIKE $1", fuzzy_name_filter));
+ }
+
+ let extensions = extension::Entity::find()
+ .filter(condition)
+ .order_by_desc(extension::Column::TotalDownloadCount)
+ .order_by_asc(extension::Column::Id)
+ .limit(Some(limit as u64))
+ .filter(
+ extension::Column::LatestVersion
+ .into_expr()
+ .eq(extension_version::Column::Version.into_expr()),
+ )
+ .inner_join(extension_version::Entity)
+ .select_also(extension_version::Entity)
+ .all(&*tx)
+ .await?;
+
+ Ok(extensions
+ .into_iter()
+ .filter_map(|(extension, latest_version)| {
+ let version = latest_version?;
+ Some(ExtensionMetadata {
+ id: extension.external_id,
+ name: extension.name,
+ version: version.version,
+ authors: version
+ .authors
+ .split(',')
+ .map(|author| author.trim().to_string())
+ .collect::<Vec<_>>(),
+ repository: version.repository,
+ published_at: version.published_at,
+ download_count: extension.total_download_count as u64,
+ })
+ })
+ .collect())
+ })
+ .await
+ }
+
+ pub async fn get_known_extension_versions<'a>(&self) -> Result<HashMap<String, Vec<String>>> {
+ self.transaction(|tx| async move {
+ let mut extension_external_ids_by_id = HashMap::default();
+
+ let mut rows = extension::Entity::find().stream(&*tx).await?;
+ while let Some(row) = rows.next().await {
+ let row = row?;
+ extension_external_ids_by_id.insert(row.id, row.external_id);
+ }
+ drop(rows);
+
+ let mut known_versions_by_extension_id: HashMap<String, Vec<String>> =
+ HashMap::default();
+ let mut rows = extension_version::Entity::find().stream(&*tx).await?;
+ while let Some(row) = rows.next().await {
+ let row = row?;
+
+ let Some(extension_id) = extension_external_ids_by_id.get(&row.extension_id) else {
+ continue;
+ };
+
+ let versions = known_versions_by_extension_id
+ .entry(extension_id.clone())
+ .or_default();
+ if let Err(ix) = versions.binary_search(&row.version) {
+ versions.insert(ix, row.version);
+ }
+ }
+ drop(rows);
+
+ Ok(known_versions_by_extension_id)
+ })
+ .await
+ }
+
+ pub async fn insert_extension_versions(
+ &self,
+ versions_by_extension_id: &HashMap<&str, Vec<NewExtensionVersion>>,
+ ) -> Result<()> {
+ self.transaction(|tx| async move {
+ for (external_id, versions) in versions_by_extension_id {
+ if versions.is_empty() {
+ continue;
+ }
+
+ let latest_version = versions
+ .iter()
+ .max_by_key(|version| &version.version)
+ .unwrap();
+
+ let insert = extension::Entity::insert(extension::ActiveModel {
+ name: ActiveValue::Set(latest_version.name.clone()),
+ external_id: ActiveValue::Set(external_id.to_string()),
+ id: ActiveValue::NotSet,
+ latest_version: ActiveValue::Set(latest_version.version.to_string()),
+ total_download_count: ActiveValue::NotSet,
+ })
+ .on_conflict(
+ OnConflict::columns([extension::Column::ExternalId])
+ .update_column(extension::Column::ExternalId)
+ .to_owned(),
+ );
+
+ let extension = if tx.support_returning() {
+ insert.exec_with_returning(&*tx).await?
+ } else {
+ // Sqlite
+ insert.exec_without_returning(&*tx).await?;
+ extension::Entity::find()
+ .filter(extension::Column::ExternalId.eq(*external_id))
+ .one(&*tx)
+ .await?
+ .ok_or_else(|| anyhow!("failed to insert extension"))?
+ };
+
+ extension_version::Entity::insert_many(versions.iter().map(|version| {
+ extension_version::ActiveModel {
+ extension_id: ActiveValue::Set(extension.id),
+ published_at: ActiveValue::Set(version.published_at),
+ version: ActiveValue::Set(version.version.to_string()),
+ authors: ActiveValue::Set(version.authors.join(", ")),
+ repository: ActiveValue::Set(version.repository.clone()),
+ description: ActiveValue::Set(version.description.clone()),
+ download_count: ActiveValue::NotSet,
+ }
+ }))
+ .on_conflict(OnConflict::new().do_nothing().to_owned())
+ .exec_without_returning(&*tx)
+ .await?;
+
+ if let Ok(db_version) = semver::Version::parse(&extension.latest_version) {
+ if db_version >= latest_version.version {
+ continue;
+ }
+ }
+
+ let mut extension = extension.into_active_model();
+ extension.latest_version = ActiveValue::Set(latest_version.version.to_string());
+ extension.name = ActiveValue::set(latest_version.name.clone());
+ extension::Entity::update(extension).exec(&*tx).await?;
+ }
+
+ Ok(())
+ })
+ .await
+ }
+
+ pub async fn record_extension_download(&self, extension: &str, version: &str) -> Result<bool> {
+ self.transaction(|tx| async move {
+ #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+ enum QueryId {
+ Id,
+ }
+
+ let extension_id: Option<ExtensionId> = extension::Entity::find()
+ .filter(extension::Column::ExternalId.eq(extension))
+ .select_only()
+ .column(extension::Column::Id)
+ .into_values::<_, QueryId>()
+ .one(&*tx)
+ .await?;
+ let Some(extension_id) = extension_id else {
+ return Ok(false);
+ };
+
+ extension_version::Entity::update_many()
+ .col_expr(
+ extension_version::Column::DownloadCount,
+ extension_version::Column::DownloadCount.into_expr().add(1),
+ )
+ .filter(
+ extension_version::Column::ExtensionId
+ .eq(extension_id)
+ .and(extension_version::Column::Version.eq(version)),
+ )
+ .exec(&*tx)
+ .await?;
+
+ extension::Entity::update_many()
+ .col_expr(
+ extension::Column::TotalDownloadCount,
+ extension::Column::TotalDownloadCount.into_expr().add(1),
+ )
+ .filter(extension::Column::Id.eq(extension_id))
+ .exec(&*tx)
+ .await?;
+
+ Ok(true)
+ })
+ .await
+ }
+}
@@ -10,6 +10,8 @@ pub mod channel_message;
pub mod channel_message_mention;
pub mod contact;
pub mod contributor;
+pub mod extension;
+pub mod extension_version;
pub mod feature_flag;
pub mod follower;
pub mod language_server;
@@ -0,0 +1,27 @@
+use crate::db::ExtensionId;
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
+#[sea_orm(table_name = "extensions")]
+pub struct Model {
+ #[sea_orm(primary_key)]
+ pub id: ExtensionId,
+ pub external_id: String,
+ pub name: String,
+ pub latest_version: String,
+ pub total_download_count: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+ #[sea_orm(has_one = "super::extension_version::Entity")]
+ LatestVersion,
+}
+
+impl Related<super::extension_version::Entity> for Entity {
+ fn to() -> RelationDef {
+ Relation::LatestVersion.def()
+ }
+}
+
+impl ActiveModelBehavior for ActiveModel {}
@@ -0,0 +1,36 @@
+use crate::db::ExtensionId;
+use sea_orm::entity::prelude::*;
+use time::PrimitiveDateTime;
+
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
+#[sea_orm(table_name = "extension_versions")]
+pub struct Model {
+ #[sea_orm(primary_key)]
+ pub extension_id: ExtensionId,
+ #[sea_orm(primary_key)]
+ pub version: String,
+ pub published_at: PrimitiveDateTime,
+ pub authors: String,
+ pub repository: String,
+ pub description: String,
+ pub download_count: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+ #[sea_orm(
+ belongs_to = "super::extension::Entity",
+ from = "Column::ExtensionId",
+ to = "super::extension::Column::Id"
+ on_condition = r#"super::extension::Column::LatestVersion.into_expr().eq(Column::Version.into_expr())"#
+ )]
+ Extension,
+}
+
+impl Related<super::extension::Entity> for Entity {
+ fn to() -> RelationDef {
+ Relation::Extension.def()
+ }
+}
+
+impl ActiveModelBehavior for ActiveModel {}
@@ -2,6 +2,7 @@ mod buffer_tests;
mod channel_tests;
mod contributor_tests;
mod db_tests;
+mod extension_tests;
mod feature_flag_tests;
mod message_tests;
@@ -0,0 +1,219 @@
+use super::Database;
+use crate::{
+ db::{ExtensionMetadata, NewExtensionVersion},
+ test_both_dbs,
+};
+use std::sync::Arc;
+use time::{OffsetDateTime, PrimitiveDateTime};
+
+test_both_dbs!(
+ test_extensions,
+ test_extensions_postgres,
+ test_extensions_sqlite
+);
+
+async fn test_extensions(db: &Arc<Database>) {
+ let versions = db.get_known_extension_versions().await.unwrap();
+ assert!(versions.is_empty());
+
+ let extensions = db.get_extensions(None, 5).await.unwrap();
+ assert!(extensions.is_empty());
+
+ let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
+ let t0 = PrimitiveDateTime::new(t0.date(), t0.time());
+
+ db.insert_extension_versions(
+ &[
+ (
+ "ext1",
+ vec![
+ NewExtensionVersion {
+ name: "Extension 1".into(),
+ version: semver::Version::parse("0.0.1").unwrap(),
+ description: "an extension".into(),
+ authors: vec!["max".into()],
+ repository: "ext1/repo".into(),
+ published_at: t0,
+ },
+ NewExtensionVersion {
+ name: "Extension One".into(),
+ version: semver::Version::parse("0.0.2").unwrap(),
+ description: "a good extension".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ repository: "ext1/repo".into(),
+ published_at: t0,
+ },
+ ],
+ ),
+ (
+ "ext2",
+ vec![NewExtensionVersion {
+ name: "Extension Two".into(),
+ version: semver::Version::parse("0.2.0").unwrap(),
+ description: "a great extension".into(),
+ authors: vec!["marshall".into()],
+ repository: "ext2/repo".into(),
+ published_at: t0,
+ }],
+ ),
+ ]
+ .into_iter()
+ .collect(),
+ )
+ .await
+ .unwrap();
+
+ let versions = db.get_known_extension_versions().await.unwrap();
+ assert_eq!(
+ versions,
+ [
+ ("ext1".into(), vec!["0.0.1".into(), "0.0.2".into()]),
+ ("ext2".into(), vec!["0.2.0".into()])
+ ]
+ .into_iter()
+ .collect()
+ );
+
+ // The latest version of each extension is returned.
+ let extensions = db.get_extensions(None, 5).await.unwrap();
+ assert_eq!(
+ extensions,
+ &[
+ ExtensionMetadata {
+ id: "ext1".into(),
+ name: "Extension One".into(),
+ version: "0.0.2".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ repository: "ext1/repo".into(),
+ published_at: t0,
+ download_count: 0,
+ },
+ ExtensionMetadata {
+ id: "ext2".into(),
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ repository: "ext2/repo".into(),
+ published_at: t0,
+ download_count: 0
+ },
+ ]
+ );
+
+ // Record extensions being downloaded.
+ for _ in 0..7 {
+ assert!(db.record_extension_download("ext2", "0.0.2").await.unwrap());
+ }
+
+ for _ in 0..3 {
+ assert!(db.record_extension_download("ext1", "0.0.1").await.unwrap());
+ }
+
+ for _ in 0..2 {
+ assert!(db.record_extension_download("ext1", "0.0.2").await.unwrap());
+ }
+
+ // Record download returns false if the extension does not exist.
+ assert!(!db
+ .record_extension_download("no-such-extension", "0.0.2")
+ .await
+ .unwrap());
+
+ // Extensions are returned in descending order of total downloads.
+ let extensions = db.get_extensions(None, 5).await.unwrap();
+ assert_eq!(
+ extensions,
+ &[
+ ExtensionMetadata {
+ id: "ext2".into(),
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ repository: "ext2/repo".into(),
+ published_at: t0,
+ download_count: 7
+ },
+ ExtensionMetadata {
+ id: "ext1".into(),
+ name: "Extension One".into(),
+ version: "0.0.2".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ repository: "ext1/repo".into(),
+ published_at: t0,
+ download_count: 5,
+ },
+ ]
+ );
+
+ // Add more extensions, including a new version of `ext1`, and backfilling
+ // an older version of `ext2`.
+ db.insert_extension_versions(
+ &[
+ (
+ "ext1",
+ vec![NewExtensionVersion {
+ name: "Extension One".into(),
+ version: semver::Version::parse("0.0.3").unwrap(),
+ description: "a real good extension".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ repository: "ext1/repo".into(),
+ published_at: t0,
+ }],
+ ),
+ (
+ "ext2",
+ vec![NewExtensionVersion {
+ name: "Extension Two".into(),
+ version: semver::Version::parse("0.1.0").unwrap(),
+ description: "an old extension".into(),
+ authors: vec!["marshall".into()],
+ repository: "ext2/repo".into(),
+ published_at: t0,
+ }],
+ ),
+ ]
+ .into_iter()
+ .collect(),
+ )
+ .await
+ .unwrap();
+
+ let versions = db.get_known_extension_versions().await.unwrap();
+ assert_eq!(
+ versions,
+ [
+ (
+ "ext1".into(),
+ vec!["0.0.1".into(), "0.0.2".into(), "0.0.3".into()]
+ ),
+ ("ext2".into(), vec!["0.1.0".into(), "0.2.0".into()])
+ ]
+ .into_iter()
+ .collect()
+ );
+
+ let extensions = db.get_extensions(None, 5).await.unwrap();
+ assert_eq!(
+ extensions,
+ &[
+ ExtensionMetadata {
+ id: "ext2".into(),
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ repository: "ext2/repo".into(),
+ published_at: t0,
+ download_count: 7
+ },
+ ExtensionMetadata {
+ id: "ext1".into(),
+ name: "Extension One".into(),
+ version: "0.0.3".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ repository: "ext1/repo".into(),
+ published_at: t0,
+ download_count: 5,
+ },
+ ]
+ );
+}
@@ -3,7 +3,8 @@ use std::fs;
pub fn load_dotenv() -> anyhow::Result<()> {
let env: toml::map::Map<String, toml::Value> = toml::de::from_str(
- &fs::read_to_string("./.env.toml").map_err(|_| anyhow!("no .env.toml file found"))?,
+ &fs::read_to_string("./crates/collab/.env.toml")
+ .map_err(|_| anyhow!("no .env.toml file found"))?,
)?;
for (key, value) in env {
@@ -8,11 +8,14 @@ pub mod rpc;
#[cfg(test)]
mod tests;
+use anyhow::anyhow;
+use aws_config::{BehaviorVersion, Region};
use axum::{http::StatusCode, response::IntoResponse};
use db::Database;
use executor::Executor;
use serde::Deserialize;
use std::{path::PathBuf, sync::Arc};
+use util::ResultExt;
pub type Result<T, E = Error> = std::result::Result<T, E>;
@@ -100,6 +103,11 @@ pub struct Config {
pub live_kit_secret: Option<String>,
pub rust_log: Option<String>,
pub log_json: Option<bool>,
+ pub blob_store_url: Option<String>,
+ pub blob_store_region: Option<String>,
+ pub blob_store_access_key: Option<String>,
+ pub blob_store_secret_key: Option<String>,
+ pub blob_store_bucket: Option<String>,
pub zed_environment: Arc<str>,
}
@@ -118,6 +126,7 @@ pub struct MigrateConfig {
pub struct AppState {
pub db: Arc<Database>,
pub live_kit_client: Option<Arc<dyn live_kit_server::api::Client>>,
+ pub blob_store_client: Option<aws_sdk_s3::Client>,
pub config: Config,
}
@@ -146,8 +155,44 @@ impl AppState {
let this = Self {
db: Arc::new(db),
live_kit_client,
+ blob_store_client: build_blob_store_client(&config).await.log_err(),
config,
};
Ok(Arc::new(this))
}
}
+
+async fn build_blob_store_client(config: &Config) -> anyhow::Result<aws_sdk_s3::Client> {
+ let keys = aws_sdk_s3::config::Credentials::new(
+ config
+ .blob_store_access_key
+ .clone()
+ .ok_or_else(|| anyhow!("missing blob_store_access_key"))?,
+ config
+ .blob_store_secret_key
+ .clone()
+ .ok_or_else(|| anyhow!("missing blob_store_secret_key"))?,
+ None,
+ None,
+ "env",
+ );
+
+ let s3_config = aws_config::defaults(BehaviorVersion::latest())
+ .endpoint_url(
+ config
+ .blob_store_url
+ .as_ref()
+ .ok_or_else(|| anyhow!("missing blob_store_url"))?,
+ )
+ .region(Region::new(
+ config
+ .blob_store_region
+ .clone()
+ .ok_or_else(|| anyhow!("missing blob_store_region"))?,
+ ))
+ .credentials_provider(keys)
+ .load()
+ .await;
+
+ Ok(aws_sdk_s3::Client::new(&s3_config))
+}
@@ -1,6 +1,9 @@
use anyhow::anyhow;
use axum::{routing::get, Extension, Router};
-use collab::{db, env, executor::Executor, AppState, Config, MigrateConfig, Result};
+use collab::{
+ api::fetch_extensions_from_blob_store_periodically, db, env, executor::Executor, AppState,
+ Config, MigrateConfig, Result,
+};
use db::Database;
use std::{
env::args,
@@ -50,6 +53,8 @@ async fn main() -> Result<()> {
let rpc_server = collab::rpc::Server::new(epoch, state.clone(), Executor::Production);
rpc_server.start().await?;
+ fetch_extensions_from_blob_store_periodically(state.clone(), Executor::Production);
+
let app = collab::api::routes(rpc_server.clone(), state.clone())
.merge(collab::rpc::routes(rpc_server.clone()))
.merge(
@@ -479,6 +479,7 @@ impl TestServer {
Arc::new(AppState {
db: test_db.db().clone(),
live_kit_client: Some(Arc::new(fake_server.create_api_client())),
+ blob_store_client: None,
config: Config {
http_port: 0,
database_url: "".into(),
@@ -491,6 +492,11 @@ impl TestServer {
rust_log: None,
log_json: None,
zed_environment: "test".into(),
+ blob_store_url: None,
+ blob_store_region: None,
+ blob_store_access_key: None,
+ blob_store_secret_key: None,
+ blob_store_bucket: None,
},
})
}
@@ -3,6 +3,10 @@
echo "installing foreman..."
which foreman > /dev/null || brew install foreman
+echo "installing minio..."
+which minio > /dev/null || brew install minio/stable/minio
+mkdir -p .blob_store/the-extensions-bucket
+
echo "creating database..."
script/sqlx database create
@@ -1,5 +1,4 @@
#!/bin/bash
set -e
-cd crates/collab
cargo run --quiet --package=collab --features seed-support --bin seed -- $@