Detailed changes
@@ -4,7 +4,9 @@
package db
-import "database/sql"
+import (
+ "database/sql"
+)
// AddRelease adds a release for a project with a given URL to the database
@@ -14,16 +16,16 @@ import "database/sql"
// UpsertRelease adds or updates a release for a project with a given URL in the
// database
-func UpsertRelease(db *sql.DB, projectURL, releaseURL, tag, content, date string) error {
- _, err := db.Exec(`INSERT INTO releases (project_url, release_url, tag, content, date)
- VALUES (?, ?, ?, ?, ?)
- ON CONFLICT(release_url) DO
+func UpsertRelease(db *sql.DB, id, projectURL, releaseURL, tag, content, date string) error {
+ _, err := db.Exec(`INSERT INTO releases (id, project_url, release_url, tag, content, date)
+ VALUES (?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO
UPDATE SET
release_url = excluded.release_url,
content = excluded.content,
tag = excluded.tag,
content = excluded.content,
- date = excluded.date;`, projectURL, releaseURL, tag, content, date)
+ date = excluded.date;`, id, projectURL, releaseURL, tag, content, date)
return err
}
@@ -4,43 +4,44 @@
-- Create table of users with username, password hash, salt, and creation
-- timestamp
-CREATE TABLE users (
- username VARCHAR(255) NOT NULL,
- hash VARCHAR(255) NOT NULL,
- salt VARCHAR(255) NOT NULL,
- created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (username)
+CREATE TABLE users
+(
+ username TEXT NOT NULL PRIMARY KEY,
+ hash TEXT NOT NULL,
+ salt TEXT NOT NULL,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create table of sessions with session GUID, username, and timestamp of when
-- the session was created
-CREATE TABLE sessions (
- token VARCHAR(255) NOT NULL,
- username VARCHAR(255) NOT NULL,
- expires TIMESTAMP NOT NULL,
- created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (token)
+CREATE TABLE sessions
+(
+ token TEXT NOT NULL,
+ username TEXT NOT NULL,
+ expires TIMESTAMP NOT NULL,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create table of tracked projects with URL, name, forge, running version, and
-- timestamp of when the project was added
-CREATE TABLE projects (
- url VARCHAR(255) NOT NULL,
- name VARCHAR(255) NOT NULL,
- forge VARCHAR(255) NOT NULL,
- version VARCHAR(255) NOT NULL,
- created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (url)
+CREATE TABLE projects
+(
+ url TEXT NOT NULL,
+ name TEXT NOT NULL,
+ forge TEXT NOT NULL,
+ version TEXT NOT NULL,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create table of project releases with the project URL and the release tags,
-- contents, URLs, and dates
-CREATE TABLE releases (
- project_url VARCHAR(255) NOT NULL,
- release_url VARCHAR(255) NOT NULL,
- tag VARCHAR(255) NOT NULL,
- content TEXT NOT NULL,
- date TIMESTAMP NOT NULL,
- created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (release_url)
+CREATE TABLE releases
+(
+ id TEXT NOT NULL PRIMARY KEY,
+ project_url TEXT NOT NULL,
+ release_url TEXT NOT NULL,
+ tag TEXT NOT NULL,
+ content TEXT NOT NULL,
+ date TIMESTAMP NOT NULL,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
@@ -7,6 +7,7 @@ package git
import (
"errors"
"fmt"
+ "io"
"net/url"
"os"
"sort"
@@ -64,31 +65,39 @@ func GetReleases(gitURI, forge string) ([]Release, error) {
releases := make([]Release, 0)
err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error {
- obj, err := r.TagObject(tagRef.Hash())
- switch {
- case errors.Is(err, plumbing.ErrObjectNotFound):
- // This is a lightweight tag, not an annotated tag, skip it
- return nil
- case err == nil:
- tagURL := ""
- tagName := bmStrict.Sanitize(tagRef.Name().Short())
- switch forge {
- case "sourcehut":
- tagURL = "https://" + httpURI + "/refs/" + tagName
- case "gitlab":
- tagURL = "https://" + httpURI + "/-/releases/" + tagName
- default:
- tagURL = ""
+ tagObj, err := r.TagObject(tagRef.Hash())
+
+ var message string
+ var date time.Time
+ if errors.Is(err, plumbing.ErrObjectNotFound) {
+ commitTag, err := r.CommitObject(tagRef.Hash())
+ if err != nil {
+ return err
}
- releases = append(releases, Release{
- Tag: tagName,
- Content: bmUGC.Sanitize(obj.Message),
- URL: tagURL,
- Date: obj.Tagger.When,
- })
+ message = commitTag.Message
+ date = commitTag.Committer.When
+ } else {
+ message = tagObj.Message
+ date = tagObj.Tagger.When
+ }
+
+ tagURL := ""
+ tagName := bmStrict.Sanitize(tagRef.Name().Short())
+ switch forge {
+ case "sourcehut":
+ tagURL = "https://" + httpURI + "/refs/" + tagName
+ case "gitlab":
+ tagURL = "https://" + httpURI + "/-/releases/" + tagName
default:
- return err
+ tagURL = ""
}
+
+ releases = append(releases, Release{
+ Tag: tagName,
+ Content: bmUGC.Sanitize(message),
+ URL: tagURL,
+ Date: date,
+ })
return nil
})
if err != nil {
@@ -139,9 +148,48 @@ func RemoveRepo(url string) (err error) {
return err
}
err = os.RemoveAll(path)
+ if err != nil {
+ return err
+ }
+
+ // TODO: Check whether the two parent directories are empty and remove them if
+ // so
+ for i := 0; i < 2; i++ {
+ path = strings.TrimSuffix(path, "/")
+ if path == "data" {
+ break
+ }
+ empty, err := dirEmpty(path)
+ if err != nil {
+ return err
+ }
+ if empty {
+ err = os.Remove(path)
+ if err != nil {
+ return err
+ }
+ }
+ path = path[:strings.LastIndex(path, "/")]
+ }
+
return err
}
+// dirEmpty checks if a directory is empty.
+func dirEmpty(name string) (empty bool, err error) {
+ f, err := os.Open(name)
+ if err != nil {
+ return false, err
+ }
+ defer f.Close()
+
+ _, err = f.Readdirnames(1)
+ if err == io.EOF {
+ return true, nil
+ }
+ return false, err
+}
+
// stringifyRepo accepts a repository URI string and the corresponding local
// filesystem path, whether the URI is HTTP, HTTPS, or SSH.
func stringifyRepo(url string) (path string, err error) {
@@ -5,6 +5,7 @@
package project
import (
+ "crypto/sha256"
"database/sql"
"fmt"
"log"
@@ -40,7 +41,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
}
if len(ret) == 0 {
- return fetchReleases(proj)
+ return fetchReleases(dbConn, proj)
}
for _, row := range ret {
@@ -58,7 +59,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
}
// fetchReleases fetches releases from a project's forge given its URI
-func fetchReleases(p Project) (Project, error) {
+func fetchReleases(dbConn *sql.DB, p Project) (Project, error) {
var err error
switch p.Forge {
case "github", "gitea", "forgejo":
@@ -74,6 +75,11 @@ func fetchReleases(p Project) (Project, error) {
URL: release.URL,
Date: release.Date,
})
+ err = upsert(dbConn, p.URL, p.Releases)
+ if err != nil {
+ log.Printf("Error upserting release: %v", err)
+ return p, err
+ }
}
default:
gitReleases, err := git.GetReleases(p.URL, p.Forge)
@@ -87,6 +93,11 @@ func fetchReleases(p Project) (Project, error) {
URL: release.URL,
Date: release.Date,
})
+ err = upsert(dbConn, p.URL, p.Releases)
+ if err != nil {
+ log.Printf("Error upserting release: %v", err)
+ return p, err
+ }
}
}
sort.Slice(p.Releases, func(i, j int) bool {
@@ -95,6 +106,21 @@ func fetchReleases(p Project) (Project, error) {
return p, err
}
+// upsert updates or inserts a project release into the database
+func upsert(dbConn *sql.DB, url string, releases []Release) error {
+ for _, release := range releases {
+ date := release.Date.Format("2006-01-02 15:04:05")
+ idByte := sha256.Sum256([]byte(url + release.URL + release.Tag + date))
+ id := fmt.Sprintf("%x", idByte)
+ err := db.UpsertRelease(dbConn, id, url, release.URL, release.Tag, release.Content, date)
+ if err != nil {
+ log.Printf("Error upserting release: %v", err)
+ return err
+ }
+ }
+ return nil
+}
+
func Track(dbConn *sql.DB, manualRefresh *chan struct{}, name, url, forge, release string) {
err := db.UpsertProject(dbConn, url, name, forge, release)
if err != nil {
@@ -126,7 +152,7 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
fmt.Println("Error getting projects:", err)
}
for i, p := range projectsList {
- p, err := fetchReleases(p)
+ p, err := fetchReleases(dbConn, p)
if err != nil {
fmt.Println(err)
continue
@@ -137,12 +163,10 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
return strings.ToLower(projectsList[i].Name) < strings.ToLower(projectsList[j].Name)
})
for i := range projectsList {
- for j := range projectsList[i].Releases {
- err = db.UpsertRelease(dbConn, projectsList[i].URL, projectsList[i].Releases[j].URL, projectsList[i].Releases[j].Tag, projectsList[i].Releases[j].Content, projectsList[i].Releases[j].Date.Format("2006-01-02 15:04:05"))
- if err != nil {
- fmt.Println("Error upserting release:", err)
- continue
- }
+ err = upsert(dbConn, projectsList[i].URL, projectsList[i].Releases)
+ if err != nil {
+ fmt.Println("Error upserting release:", err)
+ continue
}
}
return projectsList
@@ -8,7 +8,6 @@ import (
"database/sql"
"embed"
"fmt"
- "git.sr.ht/~amolith/willow/users"
"io"
"net/http"
"net/url"
@@ -17,6 +16,8 @@ import (
"text/template"
"time"
+ "git.sr.ht/~amolith/willow/users"
+
"git.sr.ht/~amolith/willow/project"
"github.com/microcosm-cc/bluemonday"
)