@@ -30,7 +30,7 @@ import (
// GitRoute is a route for git services.
type GitRoute struct {
- method string
+ method []string
pattern *regexp.Regexp
handler http.HandlerFunc
}
@@ -43,19 +43,25 @@ func (g GitRoute) Match(r *http.Request) *http.Request {
ctx := r.Context()
cfg := config.FromContext(ctx)
if m := re.FindStringSubmatch(r.URL.Path); m != nil {
+ // This finds the Git objects & packs filenames in the URL.
file := strings.Replace(r.URL.Path, m[1]+"/", "", 1)
- repo := utils.SanitizeRepo(m[1]) + ".git"
+ repo := utils.SanitizeRepo(m[1])
- var service git.Service
switch {
case strings.HasSuffix(r.URL.Path, git.UploadPackService.String()):
- service = git.UploadPackService
+ ctx = context.WithValue(ctx, pattern.Variable("service"), git.UploadPackService.String())
case strings.HasSuffix(r.URL.Path, git.ReceivePackService.String()):
- service = git.ReceivePackService
+ ctx = context.WithValue(ctx, pattern.Variable("service"), git.ReceivePackService.String())
+ case len(m) > 1:
+ // XXX: right now, the only pattern that captures more than one group
+ // is the Git LFS basic upload/download handler. This captures the LFS
+ // object Oid.
+ // See the Git LFS basic handler down below.
+ // TODO: make this more generic.
+ ctx = context.WithValue(ctx, pattern.Variable("oid"), m[2])
}
- ctx = context.WithValue(ctx, pattern.Variable("service"), service.String())
- ctx = context.WithValue(ctx, pattern.Variable("dir"), filepath.Join(cfg.DataPath, "repos", repo))
+ ctx = context.WithValue(ctx, pattern.Variable("dir"), filepath.Join(cfg.DataPath, "repos", repo+".git"))
ctx = context.WithValue(ctx, pattern.Variable("repo"), repo)
ctx = context.WithValue(ctx, pattern.Variable("file"), file)
@@ -67,7 +73,15 @@ func (g GitRoute) Match(r *http.Request) *http.Request {
// ServeHTTP implements http.Handler.
func (g GitRoute) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- if r.Method != g.method {
+ var hasMethod bool
+ for _, m := range g.method {
+ if m == r.Method {
+ hasMethod = true
+ break
+ }
+ }
+
+ if !hasMethod {
renderMethodNotAllowed(w, r)
return
}
@@ -93,76 +107,90 @@ var (
}, []string{"repo", "file"})
)
-func gitRoutes() []Route {
- routes := make([]Route, 0)
-
+var gitRoutes = []GitRoute{
// Git services
// These routes don't handle authentication/authorization.
// This is handled through wrapping the handlers for each route.
// See below (withAccess).
- // TODO: add lfs support
- for _, route := range []GitRoute{
- {
- pattern: regexp.MustCompile("(.*?)/git-upload-pack$"),
- method: http.MethodPost,
- handler: serviceRpc,
- },
- {
- pattern: regexp.MustCompile("(.*?)/git-receive-pack$"),
- method: http.MethodPost,
- handler: serviceRpc,
- },
- {
- pattern: regexp.MustCompile("(.*?)/info/refs$"),
- method: http.MethodGet,
- handler: getInfoRefs,
- },
- {
- pattern: regexp.MustCompile("(.*?)/HEAD$"),
- method: http.MethodGet,
- handler: getTextFile,
- },
- {
- pattern: regexp.MustCompile("(.*?)/objects/info/alternates$"),
- method: http.MethodGet,
- handler: getTextFile,
- },
- {
- pattern: regexp.MustCompile("(.*?)/objects/info/http-alternates$"),
- method: http.MethodGet,
- handler: getTextFile,
- },
- {
- pattern: regexp.MustCompile("(.*?)/objects/info/packs$"),
- method: http.MethodGet,
- handler: getInfoPacks,
- },
- {
- pattern: regexp.MustCompile("(.*?)/objects/info/[^/]*$"),
- method: http.MethodGet,
- handler: getTextFile,
- },
- {
- pattern: regexp.MustCompile("(.*?)/objects/[0-9a-f]{2}/[0-9a-f]{38}$"),
- method: http.MethodGet,
- handler: getLooseObject,
- },
- {
- pattern: regexp.MustCompile(`(.*?)/objects/pack/pack-[0-9a-f]{40}\.pack$`),
- method: http.MethodGet,
- handler: getPackFile,
- },
- {
- pattern: regexp.MustCompile(`(.*?)/objects/pack/pack-[0-9a-f]{40}\.idx$`),
- method: http.MethodGet,
- handler: getIdxFile,
- },
- } {
- route.handler = withAccess(route.handler)
- routes = append(routes, route)
- }
-
- return routes
+ {
+ pattern: regexp.MustCompile("(.*?)/git-upload-pack$"),
+ method: []string{http.MethodPost},
+ handler: serviceRpc,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/git-receive-pack$"),
+ method: []string{http.MethodPost},
+ handler: serviceRpc,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/info/refs$"),
+ method: []string{http.MethodGet},
+ handler: getInfoRefs,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/HEAD$"),
+ method: []string{http.MethodGet},
+ handler: getTextFile,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/objects/info/alternates$"),
+ method: []string{http.MethodGet},
+ handler: getTextFile,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/objects/info/http-alternates$"),
+ method: []string{http.MethodGet},
+ handler: getTextFile,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/objects/info/packs$"),
+ method: []string{http.MethodGet},
+ handler: getInfoPacks,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/objects/info/[^/]*$"),
+ method: []string{http.MethodGet},
+ handler: getTextFile,
+ },
+ {
+ pattern: regexp.MustCompile("(.*?)/objects/[0-9a-f]{2}/[0-9a-f]{38}$"),
+ method: []string{http.MethodGet},
+ handler: getLooseObject,
+ },
+ {
+ pattern: regexp.MustCompile(`(.*?)/objects/pack/pack-[0-9a-f]{40}\.pack$`),
+ method: []string{http.MethodGet},
+ handler: getPackFile,
+ },
+ {
+ pattern: regexp.MustCompile(`(.*?)/objects/pack/pack-[0-9a-f]{40}\.idx$`),
+ method: []string{http.MethodGet},
+ handler: getIdxFile,
+ },
+ // Git LFS
+ {
+ pattern: regexp.MustCompile(`(.*?)/info/lfs/objects/batch$`),
+ method: []string{http.MethodPost},
+ handler: serviceLfsBatch,
+ },
+ {
+ // Git LFS basic object handler
+ pattern: regexp.MustCompile(`(.*?)/info/lfs/objects/basic/([0-9a-f]{64})$`),
+ method: []string{http.MethodGet, http.MethodPut},
+ handler: serviceLfsBasic,
+ },
+ {
+ pattern: regexp.MustCompile(`(.*?)/info/lfs/objects/basic/verify$`),
+ method: []string{http.MethodPost},
+ handler: serviceLfsBasicVerify,
+ },
+ // Git LFS locks
+ // TODO: implement locks
+ // {
+ // pattern: regexp.MustCompile(`(.*?)/info/lfs/locks$`),
+ // method: []string{http.MethodPost},
+ // handler: serviceLfsLocksCreate,
+ // },
}
// withAccess handles auth.
@@ -402,34 +430,32 @@ func updateServerInfo(ctx context.Context, dir string) error {
// HTTP error response handling functions
+func renderBadRequest(w http.ResponseWriter) {
+ renderStatus(http.StatusBadRequest)(w, nil)
+}
+
func renderMethodNotAllowed(w http.ResponseWriter, r *http.Request) {
if r.Proto == "HTTP/1.1" {
- w.WriteHeader(http.StatusMethodNotAllowed)
- w.Write([]byte("Method Not Allowed")) // nolint: errcheck
+ renderStatus(http.StatusMethodNotAllowed)(w, r)
} else {
- w.WriteHeader(http.StatusBadRequest)
- w.Write([]byte("Bad Request")) // nolint: errcheck
+ renderBadRequest(w)
}
}
func renderNotFound(w http.ResponseWriter) {
- w.WriteHeader(http.StatusNotFound)
- w.Write([]byte("Not Found")) // nolint: errcheck
+ renderStatus(http.StatusNotFound)(w, nil)
}
func renderUnauthorized(w http.ResponseWriter) {
- w.WriteHeader(http.StatusUnauthorized)
- w.Write([]byte("Unauthorized")) // nolint: errcheck
+ renderStatus(http.StatusUnauthorized)(w, nil)
}
func renderForbidden(w http.ResponseWriter) {
- w.WriteHeader(http.StatusForbidden)
- w.Write([]byte("Forbidden")) // nolint: errcheck
+ renderStatus(http.StatusForbidden)(w, nil)
}
func renderInternalServerError(w http.ResponseWriter) {
- w.WriteHeader(http.StatusInternalServerError)
- w.Write([]byte("Internal Server Error")) // nolint: errcheck
+ renderStatus(http.StatusInternalServerError)(w, nil)
}
// Header writing functions
@@ -0,0 +1,409 @@
+package web
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "path"
+ "path/filepath"
+ "strconv"
+
+ "github.com/charmbracelet/log"
+ "github.com/charmbracelet/soft-serve/server/backend"
+ "github.com/charmbracelet/soft-serve/server/config"
+ "github.com/charmbracelet/soft-serve/server/db"
+ "github.com/charmbracelet/soft-serve/server/lfs"
+ "github.com/charmbracelet/soft-serve/server/storage"
+ "github.com/charmbracelet/soft-serve/server/store"
+ "goji.io/pat"
+)
+
+// serviceLfsBatch handles a Git LFS batch requests.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
+// TODO: support refname & authentication
+// POST: /<repo>.git/info/lfs/objects/batch
+func serviceLfsBatch(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Content-Type") != lfs.MediaType {
+ renderNotAcceptable(w)
+ return
+ }
+
+ var batchRequest lfs.BatchRequest
+ ctx := r.Context()
+ logger := log.FromContext(ctx).WithPrefix("http.lfs")
+
+ defer r.Body.Close() // nolint: errcheck
+ if err := json.NewDecoder(r.Body).Decode(&batchRequest); err != nil {
+ logger.Errorf("error decoding json: %s", err)
+ return
+ }
+
+ // We only accept basic transfers for now
+ // Default to basic if no transfer is specified
+ if len(batchRequest.Transfers) > 0 {
+ var isBasic bool
+ for _, t := range batchRequest.Transfers {
+ if t == lfs.TransferBasic {
+ isBasic = true
+ break
+ }
+ }
+
+ if !isBasic {
+ renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
+ Message: "unsupported transfer",
+ })
+ return
+ }
+ }
+
+ be := backend.FromContext(ctx)
+ name := pat.Param(r, "repo")
+ repo, err := be.Repository(ctx, name)
+ if err != nil {
+ renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
+ Message: "repository not found",
+ })
+ return
+ }
+
+ cfg := config.FromContext(ctx)
+ dbx := db.FromContext(ctx)
+ datastore := store.FromContext(ctx)
+ // TODO: support S3 storage
+ strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs"))
+
+ baseHref := fmt.Sprintf("%s/%s/info/lfs/objects/basic", cfg.HTTP.PublicURL, name+".git")
+
+ var batchResponse lfs.BatchResponse
+ batchResponse.Transfer = lfs.TransferBasic
+ batchResponse.HashAlgo = lfs.HashAlgorithmSHA256
+
+ objects := make([]*lfs.ObjectResponse, 0, len(batchRequest.Objects))
+ // XXX: We don't support objects TTL for now, probably implement that with
+ // S3 using object "expires_at" & "expires_in"
+ switch batchRequest.Operation {
+ case lfs.OperationDownload:
+ for _, o := range batchRequest.Objects {
+ stat, err := strg.Stat(path.Join("objects", o.RelativePath()))
+ if err != nil && !errors.Is(err, fs.ErrNotExist) {
+ logger.Error("error getting object stat", "oid", o.Oid, "repo", name, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ obj, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), o.Oid)
+ if err != nil && !errors.Is(err, db.ErrRecordNotFound) {
+ logger.Error("error getting object from database", "oid", o.Oid, "repo", name, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ if stat == nil {
+ objects = append(objects, &lfs.ObjectResponse{
+ Pointer: o,
+ Error: &lfs.ObjectError{
+ Code: http.StatusNotFound,
+ Message: "object not found",
+ },
+ })
+ } else if stat.Size() != o.Size {
+ objects = append(objects, &lfs.ObjectResponse{
+ Pointer: o,
+ Error: &lfs.ObjectError{
+ Code: http.StatusUnprocessableEntity,
+ Message: "size mismatch",
+ },
+ })
+ } else if o.IsValid() {
+ objects = append(objects, &lfs.ObjectResponse{
+ Pointer: o,
+ Actions: map[string]*lfs.Link{
+ lfs.ActionDownload: {
+ Href: fmt.Sprintf("%s/%s", baseHref, o.Oid),
+ },
+ },
+ })
+
+ // If the object doesn't exist in the database, create it
+ if stat != nil && obj.ID == 0 {
+ if err := datastore.CreateLFSObject(ctx, dbx, repo.ID(), o.Oid, stat.Size()); err != nil {
+ logger.Error("error creating object in datastore", "oid", o.Oid, "repo", name, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+ }
+ } else {
+ objects = append(objects, &lfs.ObjectResponse{
+ Pointer: o,
+ Error: &lfs.ObjectError{
+ Code: http.StatusUnprocessableEntity,
+ Message: "invalid object",
+ },
+ })
+ }
+ }
+ case lfs.OperationUpload:
+ // Object upload logic happens in the "basic" API route
+ for _, o := range batchRequest.Objects {
+ if !o.IsValid() {
+ objects = append(objects, &lfs.ObjectResponse{
+ Pointer: o,
+ Error: &lfs.ObjectError{
+ Code: http.StatusUnprocessableEntity,
+ Message: "invalid object",
+ },
+ })
+ } else {
+ objects = append(objects, &lfs.ObjectResponse{
+ Pointer: o,
+ Actions: map[string]*lfs.Link{
+ lfs.ActionUpload: {
+ Href: fmt.Sprintf("%s/%s", baseHref, o.Oid),
+ },
+ // Verify uploaded objects
+ // https://github.com/git-lfs/git-lfs/blob/main/docs/api/basic-transfers.md#verification
+ lfs.ActionVerify: {
+ Href: fmt.Sprintf("%s/verify", baseHref),
+ },
+ },
+ })
+ }
+ }
+ default:
+ renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
+ Message: "unsupported operation",
+ })
+ return
+ }
+
+ batchResponse.Objects = objects
+ renderJSON(w, http.StatusOK, batchResponse)
+}
+
+// serviceLfsBasic implements Git LFS basic transfer API
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/basic-transfers.md
+func serviceLfsBasic(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodGet:
+ serviceLfsBasicDownload(w, r)
+ case http.MethodPut:
+ serviceLfsBasicUpload(w, r)
+ }
+}
+
+// GET: /<repo>.git/info/lfs/objects/basic/<oid>
+func serviceLfsBasicDownload(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
+ oid := pat.Param(r, "oid")
+ cfg := config.FromContext(ctx)
+ logger := log.FromContext(ctx).WithPrefix("http.lfs-basic")
+ strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs"))
+
+ obj, err := strg.Open(path.Join("objects", oid))
+ if err != nil {
+ logger.Error("error opening object", "oid", oid, "err", err)
+ renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
+ Message: "object not found",
+ })
+ return
+ }
+
+ stat, err := obj.Stat()
+ if err != nil {
+ logger.Error("error getting object stat", "oid", oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ defer obj.Close() // nolint: errcheck
+ if _, err := io.Copy(w, obj); err != nil {
+ logger.Error("error copying object to response", "oid", oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Length", strconv.FormatInt(stat.Size(), 10))
+ renderStatus(http.StatusOK)(w, nil)
+}
+
+// PUT: /<repo>.git/info/lfs/objects/basic/<oid>
+func serviceLfsBasicUpload(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Content-Type") != "application/octet-stream" {
+ renderJSON(w, http.StatusUnsupportedMediaType, lfs.ErrorResponse{
+ Message: "invalid content type",
+ })
+ return
+ }
+
+ ctx := r.Context()
+ oid := pat.Param(r, "oid")
+ cfg := config.FromContext(ctx)
+ be := backend.FromContext(ctx)
+ dbx := db.FromContext(ctx)
+ datastore := store.FromContext(ctx)
+ logger := log.FromContext(ctx).WithPrefix("http.lfs-basic")
+ strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs"))
+ name := pat.Param(r, "repo")
+
+ defer r.Body.Close() // nolint: errcheck
+ repo, err := be.Repository(ctx, name)
+ if err != nil {
+ renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
+ Message: "repository not found",
+ })
+ return
+ }
+
+ // NOTE: Git LFS client will retry uploading the same object if there was a
+ // partial error, so we need to skip existing objects.
+ if _, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), oid); err == nil {
+ // Object exists, skip request
+ io.Copy(io.Discard, r.Body) // nolint: errcheck
+ renderStatus(http.StatusOK)(w, nil)
+ return
+ } else if !errors.Is(err, db.ErrRecordNotFound) {
+ logger.Error("error getting object", "oid", oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ if err := strg.Put(path.Join("objects", oid), r.Body); err != nil {
+ logger.Error("error writing object", "oid", oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
+ if err != nil {
+ logger.Error("error parsing content length", "err", err)
+ renderJSON(w, http.StatusBadRequest, lfs.ErrorResponse{
+ Message: "invalid content length",
+ })
+ return
+ }
+
+ if err := datastore.CreateLFSObject(ctx, dbx, repo.ID(), oid, size); err != nil {
+ logger.Error("error creating object", "oid", oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+
+ renderStatus(http.StatusOK)(w, nil)
+}
+
+// POST: /<repo>.git/info/lfs/objects/basic/verify
+func serviceLfsBasicVerify(w http.ResponseWriter, r *http.Request) {
+ var pointer lfs.Pointer
+ ctx := r.Context()
+ logger := log.FromContext(ctx).WithPrefix("http.lfs-basic")
+ be := backend.FromContext(ctx)
+ name := pat.Param(r, "repo")
+ repo, err := be.Repository(ctx, name)
+ if err != nil {
+ renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
+ Message: "repository not found",
+ })
+ return
+ }
+
+ defer r.Body.Close() // nolint: errcheck
+ if err := json.NewDecoder(r.Body).Decode(&pointer); err != nil {
+ logger.Error("error decoding json", "err", err)
+ renderJSON(w, http.StatusBadRequest, lfs.ErrorResponse{
+ Message: "invalid json",
+ })
+ return
+ }
+
+ cfg := config.FromContext(ctx)
+ dbx := db.FromContext(ctx)
+ datastore := store.FromContext(ctx)
+ strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs"))
+ if stat, err := strg.Stat(path.Join("objects", pointer.Oid)); err == nil {
+ // Verify object is in the database.
+ if _, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), pointer.Oid); err != nil {
+ if errors.Is(err, db.ErrRecordNotFound) {
+ // Create missing object.
+ if err := datastore.CreateLFSObject(ctx, dbx, repo.ID(), pointer.Oid, stat.Size()); err != nil {
+ logger.Error("error creating object", "oid", pointer.Oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+ } else {
+ logger.Error("error getting object", "oid", pointer.Oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+ }
+
+ if pointer.IsValid() && stat.Size() == pointer.Size {
+ renderStatus(http.StatusOK)(w, nil)
+ return
+ }
+ } else if errors.Is(err, fs.ErrNotExist) {
+ renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
+ Message: "object not found",
+ })
+ return
+ } else {
+ logger.Error("error getting object", "oid", pointer.Oid, "err", err)
+ renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
+ Message: "internal server error",
+ })
+ return
+ }
+}
+
+// POST: /<repo>.git/info/lfs/objects/locks
+func serviceLfsLocksCreate(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Content-Type") != lfs.MediaType {
+ renderNotAcceptable(w)
+ return
+ }
+
+ panic("not implemented")
+}
+
+// renderJSON renders a JSON response with the given status code and value. It
+// also sets the Content-Type header to the JSON LFS media type (application/vnd.git-lfs+json).
+func renderJSON(w http.ResponseWriter, statusCode int, v interface{}) {
+ w.Header().Set("Content-Type", lfs.MediaType)
+ renderStatus(statusCode)(w, nil)
+ if err := json.NewEncoder(w).Encode(v); err != nil {
+ log.Error("error encoding json", "err", err)
+ }
+}
+
+func renderNotAcceptable(w http.ResponseWriter) {
+ renderStatus(http.StatusNotAcceptable)(w, nil)
+}
+
+func hdrLfs(w http.ResponseWriter) {
+ w.Header().Set("Content-Type", lfs.MediaType)
+ w.Header().Set("Accept", lfs.MediaType)
+}