mentos1386/zdravko
1
0
Fork 0
mirror of https://github.com/mentos1386/zdravko.git synced 2024-11-21 23:33:34 +00:00
Code Issues Projects Releases Packages Wiki Activity

feat(keyvalue): add badger as keyvalue store

Browse source 
KeyValue store will be used by Incidents, so that the functions
can decide based on history not just on the event that triggered them.
This commit is contained in:
Tine 2024-04-28 16:06:49 +02:00
parent e77764c4b9
commit bb1ba5ed58
Signed by: mentos1386
SSH key fingerprint: SHA256:MNtTsLbihYaWF8j1fkOHfkKNlnN1JQfxEU/rBU8nCGw
9 changed files with 142 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -7,6 +7,7 @@ node_modules/
# Database
zdravko.db*
zdravko_kv.db*
temporal.db*
# Keys

6
go.mod
View file

@ -3,6 +3,7 @@ module code.tjo.space/mentos1386/zdravko
go 1.21.6
require (
github.com/dgraph-io/badger/v4 v4.2.0
github.com/go-playground/validator/v10 v10.18.0
github.com/golang-jwt/jwt/v5 v5.2.0
github.com/gorilla/sessions v1.2.2
@ -16,6 +17,7 @@ require (
github.com/spf13/viper v1.18.2
github.com/temporalio/ui-server/v2 v2.23.0
go.k6.io/k6 v0.49.0
go.temporal.io/api v1.27.0
go.temporal.io/sdk v1.26.0-rc.2
go.temporal.io/server v1.22.4
golang.org/x/exp v0.0.0-20231127185646-65229373498e
@ -49,6 +51,7 @@ require (
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/coreos/go-oidc/v3 v3.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dlclark/regexp2 v1.9.0 // indirect
@ -71,10 +74,12 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/glog v1.1.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.7.0-rc.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
@ -165,7 +170,6 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.temporal.io/api v1.27.0 // indirect
go.temporal.io/version v0.3.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.0 // indirect

11
go.sum
View file

@ -89,7 +89,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20140601200337-fc41e106ee0e/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
@ -103,6 +108,7 @@ github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d h1:wi6jN5LVt/ljaBG4ue7
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@ -172,6 +178,8 @@ github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1
github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -198,6 +206,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@ -653,6 +663,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

12
internal/config/server.go
View file

@ -10,10 +10,11 @@ import (
)
type ServerConfig struct {
Port string `validate:"required"`
RootUrl string `validate:"required,url"`
DatabasePath string `validate:"required"`
SessionSecret string `validate:"required"`
Port string `validate:"required"`
RootUrl string `validate:"required,url"`
SqliteDatabasePath string `validate:"required"`
KeyValueDatabasePath string `validate:"required"`
SessionSecret string `validate:"required"`
Jwt ServerJwt `validate:"required"`
OAuth2 ServerOAuth2 `validate:"required"`
@ -47,7 +48,8 @@ func NewServerConfig() *ServerConfig {
// Set defaults
v.SetDefault("port", GetEnvOrDefault("PORT", "8000"))
v.SetDefault("rooturl", GetEnvOrDefault("ROOT_URL", "http://localhost:8000"))
v.SetDefault("databasepath", GetEnvOrDefault("DATABASE_PATH", "zdravko.db"))
v.SetDefault("sqlitedatabasepath", GetEnvOrDefault("SQLITE_DATABASE_PATH", "zdravko.db"))
v.SetDefault("keyvaluedatabasepath", GetEnvOrDefault("KEYVALUE_DATABASE_PATH", "zdravko_kv.db"))
v.SetDefault("sessionsecret", os.Getenv("SESSION_SECRET"))
v.SetDefault("temporal.uihost", GetEnvOrDefault("TEMPORAL_UI_HOST", "127.0.0.1:8223"))
v.SetDefault("temporal.serverhost", GetEnvOrDefault("TEMPORAL_SERVER_HOST", "127.0.0.1:7233"))

11
internal/handlers/handlers.go
View file

@ -4,6 +4,7 @@ import (
"log/slog"
"code.tjo.space/mentos1386/zdravko/internal/config"
"code.tjo.space/mentos1386/zdravko/internal/kv"
"code.tjo.space/mentos1386/zdravko/web/templates/components"
"github.com/gorilla/sessions"
"github.com/jmoiron/sqlx"
@ -26,20 +27,22 @@ func GetPageByTitle(pages []*components.Page, title string) *components.Page {
}
type BaseHandler struct {
db *sqlx.DB
config *config.ServerConfig
logger *slog.Logger
db *sqlx.DB
kvStore kv.KeyValueStore
config *config.ServerConfig
logger *slog.Logger
temporal client.Client
store *sessions.CookieStore
}
func NewBaseHandler(db *sqlx.DB, temporal client.Client, config *config.ServerConfig, logger *slog.Logger) *BaseHandler {
func NewBaseHandler(db *sqlx.DB, kvStore kv.KeyValueStore, temporal client.Client, config *config.ServerConfig, logger *slog.Logger) *BaseHandler {
store := sessions.NewCookieStore([]byte(config.SessionSecret))
return &BaseHandler{
db: db,
kvStore: kvStore,
config: config,
logger: logger,
temporal: temporal,

82
internal/kv/badger.go Normal file
View file

@ -0,0 +1,82 @@
package kv
import (
"time"
badger "github.com/dgraph-io/badger/v4"
"github.com/pkg/errors"
)
type BadgerKeyValueStore struct {
db *badger.DB
}
func NewBadgerKeyValueStore(path string) (*BadgerKeyValueStore, error) {
db, err := badger.Open(badger.DefaultOptions(path))
if err != nil {
return nil, errors.Wrap(err, "failed to open badger db")
}
return &BadgerKeyValueStore{db: db}, nil
}
func (b *BadgerKeyValueStore) Close() error {
return b.db.Close()
}
func (b *BadgerKeyValueStore) Set(key string, value []byte, ttl time.Duration) error {
return b.db.Update(func(txn *badger.Txn) error {
e := badger.NewEntry([]byte(key), value).WithTTL(ttl)
return txn.SetEntry(e)
})
}
func (b *BadgerKeyValueStore) Increment(key string) (int, error) {
var value int
return value, b.db.Update(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(key))
if err != nil {
return err
}
valCopy, err := item.ValueCopy(nil)
if err != nil {
return err
}
value = int(valCopy[0]) + 1
return txn.Set([]byte(key), []byte{byte(value)})
})
}
func (b *BadgerKeyValueStore) Get(key string) ([]byte, error) {
var value []byte
return value, b.db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(key))
if err != nil {
return err
}
valCopy, err := item.ValueCopy(value)
if err != nil {
return err
}
value = valCopy
return nil
})
}
func (b *BadgerKeyValueStore) Delete(key string) error {
return b.db.Update(func(txn *badger.Txn) error {
return txn.Delete([]byte(key))
})
}
func (b *BadgerKeyValueStore) Keys(prefix string) ([]string, error) {
var keys []string
return keys, b.db.View(func(txn *badger.Txn) error {
itr := txn.NewIterator(badger.DefaultIteratorOptions)
defer itr.Close()
for itr.Seek([]byte(prefix)); itr.ValidForPrefix([]byte(prefix)); itr.Next() {
item := itr.Item()
keys = append(keys, string(item.Key()))
}
return nil
})
}

13
internal/kv/kv.go Normal file
View file

@ -0,0 +1,13 @@
package kv
import "time"
type KeyValueStore interface {
Close() error
Get(key string) ([]byte, error)
Set(key string, value []byte, ttl time.Duration) error
Increment(key string) (int, error)
Delete(key string) error
Keys(prefix string) ([]string, error)
}

8
pkg/server/routes.go
View file

@ -6,6 +6,7 @@ import (
"code.tjo.space/mentos1386/zdravko/internal/config"
"code.tjo.space/mentos1386/zdravko/internal/handlers"
"code.tjo.space/mentos1386/zdravko/internal/kv"
"code.tjo.space/mentos1386/zdravko/web/static"
"github.com/jmoiron/sqlx"
"github.com/labstack/echo/v4"
@ -15,16 +16,17 @@ import (
func Routes(
e *echo.Echo,
db *sqlx.DB,
sqlDb *sqlx.DB,
kvStore kv.KeyValueStore,
temporalClient client.Client,
cfg *config.ServerConfig,
logger *slog.Logger,
) {
h := handlers.NewBaseHandler(db, temporalClient, cfg, logger)
h := handlers.NewBaseHandler(sqlDb, kvStore, temporalClient, cfg, logger)
// Health
e.GET("/health", func(c echo.Context) error {
err := db.Ping()
err := sqlDb.Ping()
if err != nil {
return err
}

15
pkg/server/server.go
View file

@ -6,10 +6,12 @@ import (
"code.tjo.space/mentos1386/zdravko/database"
"code.tjo.space/mentos1386/zdravko/internal/config"
"code.tjo.space/mentos1386/zdravko/internal/kv"
"code.tjo.space/mentos1386/zdravko/internal/temporal"
"code.tjo.space/mentos1386/zdravko/web/templates"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
"github.com/pkg/errors"
)
type Server struct {
@ -33,14 +35,19 @@ func (s *Server) Name() string {
}
func (s *Server) Start() error {
db, err := database.ConnectToDatabase(s.logger, s.cfg.DatabasePath)
sqliteDb, err := database.ConnectToDatabase(s.logger, s.cfg.SqliteDatabasePath)
if err != nil {
return err
return errors.Wrap(err, "failed to connect to sqlite")
}
temporalClient, err := temporal.ConnectServerToTemporal(s.logger, s.cfg)
if err != nil {
return err
return errors.Wrap(err, "failed to connect to temporal")
}
kvStore, err := kv.NewBadgerKeyValueStore(s.cfg.KeyValueDatabasePath)
if err != nil {
return errors.Wrap(err, "failed to open kv store")
}
s.worker = NewWorker(temporalClient, s.cfg)
@ -49,7 +56,7 @@ func (s *Server) Start() error {
s.echo.Use(middleware.Logger())
s.echo.Use(middleware.Recover())
s.echo.Use(middleware.Secure())
Routes(s.echo, db, temporalClient, s.cfg, s.logger)
Routes(s.echo, sqliteDb, kvStore, temporalClient, s.cfg, s.logger)
go func() {
if err := s.worker.Start(); err != nil {