mirror of
https://github.com/mentos1386/zdravko.git
synced 2024-11-25 00:58:07 +00:00
feat(settings/healthchecks): updating healthchecks
This commit is contained in:
parent
cca5dc6ff3
commit
a65834bd8d
11 changed files with 122 additions and 38 deletions
|
@ -14,6 +14,7 @@ Mostly just a project to test [temporal.io](https://temporal.io/).
|
||||||
- Spread workers across regions to monitor latency from different locations.
|
- Spread workers across regions to monitor latency from different locations.
|
||||||
- [x] Use [k6](https://github.com/grafana/k6) for checks, so that they can be written in javascript.
|
- [x] Use [k6](https://github.com/grafana/k6) for checks, so that they can be written in javascript.
|
||||||
- [ ] History and working home page.
|
- [ ] History and working home page.
|
||||||
|
- Kinda working atm. But look if all the data could be stored/fetched from temporal.
|
||||||
- [ ] Edit/Delete operations for healthchecks and workers.
|
- [ ] Edit/Delete operations for healthchecks and workers.
|
||||||
- [ ] CronJob Healthchecks (via webhooks).
|
- [ ] CronJob Healthchecks (via webhooks).
|
||||||
- [ ] Notifications (webhooks, slack, etc).
|
- [ ] Notifications (webhooks, slack, etc).
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"code.tjo.space/mentos1386/zdravko/web/templates/components"
|
"code.tjo.space/mentos1386/zdravko/web/templates/components"
|
||||||
"github.com/gorilla/sessions"
|
"github.com/gorilla/sessions"
|
||||||
"go.temporal.io/sdk/client"
|
"go.temporal.io/sdk/client"
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var Pages = []*components.Page{
|
var Pages = []*components.Page{
|
||||||
|
@ -25,7 +24,6 @@ func GetPageByTitle(pages []*components.Page, title string) *components.Page {
|
||||||
}
|
}
|
||||||
|
|
||||||
type BaseHandler struct {
|
type BaseHandler struct {
|
||||||
db *gorm.DB
|
|
||||||
query *query.Query
|
query *query.Query
|
||||||
config *config.ServerConfig
|
config *config.ServerConfig
|
||||||
|
|
||||||
|
@ -34,8 +32,8 @@ type BaseHandler struct {
|
||||||
store *sessions.CookieStore
|
store *sessions.CookieStore
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewBaseHandler(db *gorm.DB, q *query.Query, temporal client.Client, config *config.ServerConfig) *BaseHandler {
|
func NewBaseHandler(q *query.Query, temporal client.Client, config *config.ServerConfig) *BaseHandler {
|
||||||
store := sessions.NewCookieStore([]byte(config.SessionSecret))
|
store := sessions.NewCookieStore([]byte(config.SessionSecret))
|
||||||
|
|
||||||
return &BaseHandler{db, q, config, temporal, store}
|
return &BaseHandler{q, config, temporal, store}
|
||||||
}
|
}
|
||||||
|
|
|
@ -82,12 +82,13 @@ func (h *BaseHandler) RefreshToken(w http.ResponseWriter, r *http.Request, user
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *BaseHandler) OAuth2LoginGET(c echo.Context) error {
|
func (h *BaseHandler) OAuth2LoginGET(c echo.Context) error {
|
||||||
|
ctx := context.Background()
|
||||||
conf := newOAuth2(h.config)
|
conf := newOAuth2(h.config)
|
||||||
|
|
||||||
state := newRandomState()
|
state := newRandomState()
|
||||||
result := h.db.Create(&models.OAuth2State{State: state, Expiry: time.Now().Add(5 * time.Minute)})
|
err := h.query.OAuth2State.WithContext(ctx).Create(&models.OAuth2State{State: state, Expiry: time.Now().Add(5 * time.Minute)})
|
||||||
if result.Error != nil {
|
if err != nil {
|
||||||
return result.Error
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
url := conf.AuthCodeURL(state, oauth2.AccessTypeOffline)
|
url := conf.AuthCodeURL(state, oauth2.AccessTypeOffline)
|
||||||
|
|
|
@ -70,6 +70,46 @@ func (h *BaseHandler) SettingsHealthchecksDescribeGET(c echo.Context) error {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *BaseHandler) SettingsHealthchecksDescribePOST(c echo.Context) error {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
slug := c.Param("slug")
|
||||||
|
|
||||||
|
healthcheck, err := services.GetHealthcheck(ctx, h.query, slug)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
update := &models.Healthcheck{
|
||||||
|
Slug: healthcheck.Slug,
|
||||||
|
Name: healthcheck.Name,
|
||||||
|
Schedule: c.FormValue("schedule"),
|
||||||
|
WorkerGroups: strings.Split(c.FormValue("workergroups"), " "),
|
||||||
|
Script: c.FormValue("script"),
|
||||||
|
}
|
||||||
|
|
||||||
|
err = validator.New(validator.WithRequiredStructEnabled()).Struct(update)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = services.UpdateHealthcheck(
|
||||||
|
ctx,
|
||||||
|
h.query,
|
||||||
|
update,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = services.CreateOrUpdateHealthcheckSchedule(ctx, h.temporal, healthcheck)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.Redirect(http.StatusSeeOther, fmt.Sprintf("/settings/healthchecks/%s", slug))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *BaseHandler) SettingsHealthchecksCreateGET(c echo.Context) error {
|
func (h *BaseHandler) SettingsHealthchecksCreateGET(c echo.Context) error {
|
||||||
cc := c.(AuthenticatedContext)
|
cc := c.(AuthenticatedContext)
|
||||||
|
|
||||||
|
@ -101,14 +141,14 @@ func (h *BaseHandler) SettingsHealthchecksCreatePOST(c echo.Context) error {
|
||||||
|
|
||||||
err = services.CreateHealthcheck(
|
err = services.CreateHealthcheck(
|
||||||
ctx,
|
ctx,
|
||||||
h.db,
|
h.query,
|
||||||
healthcheckHttp,
|
healthcheckHttp,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = services.StartHealthcheck(ctx, h.temporal, healthcheckHttp)
|
err = services.CreateOrUpdateHealthcheckSchedule(ctx, h.temporal, healthcheckHttp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -113,7 +113,7 @@ func (h *BaseHandler) SettingsWorkersCreatePOST(c echo.Context) error {
|
||||||
|
|
||||||
err = services.CreateWorker(
|
err = services.CreateWorker(
|
||||||
ctx,
|
ctx,
|
||||||
h.db,
|
h.query,
|
||||||
worker,
|
worker,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -13,8 +13,19 @@ import (
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
func CreateHealthcheck(ctx context.Context, db *gorm.DB, healthcheck *models.Healthcheck) error {
|
func getScheduleId(healthcheck *models.Healthcheck, group string) string {
|
||||||
return db.WithContext(ctx).Create(healthcheck).Error
|
return "healthcheck-" + healthcheck.Slug + "-" + group
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateHealthcheck(ctx context.Context, query *query.Query, healthcheck *models.Healthcheck) error {
|
||||||
|
return query.Healthcheck.WithContext(ctx).Create(healthcheck)
|
||||||
|
}
|
||||||
|
|
||||||
|
func UpdateHealthcheck(ctx context.Context, q *query.Query, healthcheck *models.Healthcheck) error {
|
||||||
|
_, err := q.Healthcheck.WithContext(ctx).Where(
|
||||||
|
q.Healthcheck.Slug.Eq(healthcheck.Slug),
|
||||||
|
).Updates(healthcheck)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetHealthcheck(ctx context.Context, q *query.Query, slug string) (*models.Healthcheck, error) {
|
func GetHealthcheck(ctx context.Context, q *query.Query, slug string) (*models.Healthcheck, error) {
|
||||||
|
@ -34,33 +45,62 @@ func GetHealthchecks(ctx context.Context, q *query.Query) ([]*models.Healthcheck
|
||||||
).Find()
|
).Find()
|
||||||
}
|
}
|
||||||
|
|
||||||
func StartHealthcheck(ctx context.Context, t client.Client, healthcheck *models.Healthcheck) error {
|
func CreateOrUpdateHealthcheckSchedule(ctx context.Context, t client.Client, healthcheck *models.Healthcheck) error {
|
||||||
log.Println("Starting Healthcheck Workflow")
|
log.Println("Creating or Updating Healthcheck Schedule")
|
||||||
|
|
||||||
args := make([]interface{}, 0)
|
args := make([]interface{}, 0)
|
||||||
args = append(args, workflows.HealthcheckWorkflowParam{Script: healthcheck.Script, Slug: healthcheck.Slug})
|
args = append(args, workflows.HealthcheckWorkflowParam{Script: healthcheck.Script, Slug: healthcheck.Slug})
|
||||||
|
|
||||||
id := "healthcheck-" + healthcheck.Slug
|
|
||||||
|
|
||||||
fakeWorkflows := workflows.NewWorkflows(nil)
|
|
||||||
|
|
||||||
for _, group := range healthcheck.WorkerGroups {
|
for _, group := range healthcheck.WorkerGroups {
|
||||||
_, err := t.ScheduleClient().Create(ctx, client.ScheduleOptions{
|
options := client.ScheduleOptions{
|
||||||
ID: id + "-" + group,
|
ID: getScheduleId(healthcheck, group),
|
||||||
|
//SearchAttributes: map[string]interface{}{
|
||||||
|
// "worker-group": group,
|
||||||
|
// "healthcheck-slug": healthcheck.Slug,
|
||||||
|
//},
|
||||||
Spec: client.ScheduleSpec{
|
Spec: client.ScheduleSpec{
|
||||||
CronExpressions: []string{healthcheck.Schedule},
|
CronExpressions: []string{healthcheck.Schedule},
|
||||||
Jitter: time.Second * 10,
|
Jitter: time.Second * 10,
|
||||||
},
|
},
|
||||||
Action: &client.ScheduleWorkflowAction{
|
Action: &client.ScheduleWorkflowAction{
|
||||||
ID: id + "-" + group,
|
ID: getScheduleId(healthcheck, group),
|
||||||
Workflow: fakeWorkflows.HealthcheckWorkflowDefinition,
|
Workflow: workflows.NewWorkflows(nil).HealthcheckWorkflowDefinition,
|
||||||
Args: args,
|
Args: args,
|
||||||
TaskQueue: group,
|
TaskQueue: group,
|
||||||
RetryPolicy: &temporal.RetryPolicy{
|
RetryPolicy: &temporal.RetryPolicy{
|
||||||
MaximumAttempts: 3,
|
MaximumAttempts: 3,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
|
||||||
|
schedule := t.ScheduleClient().GetHandle(ctx, getScheduleId(healthcheck, group))
|
||||||
|
|
||||||
|
// If exists, we update
|
||||||
|
_, err := schedule.Describe(ctx)
|
||||||
|
if err == nil {
|
||||||
|
err = schedule.Update(ctx, client.ScheduleUpdateOptions{
|
||||||
|
DoUpdate: func(input client.ScheduleUpdateInput) (*client.ScheduleUpdate, error) {
|
||||||
|
return &client.ScheduleUpdate{
|
||||||
|
Schedule: &client.Schedule{
|
||||||
|
Spec: &options.Spec,
|
||||||
|
Action: options.Action,
|
||||||
|
Policy: input.Description.Schedule.Policy,
|
||||||
|
State: input.Description.Schedule.State,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
schedule, err = t.ScheduleClient().Create(ctx, options)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = schedule.Trigger(ctx, client.ScheduleTriggerOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,11 +6,10 @@ import (
|
||||||
|
|
||||||
"code.tjo.space/mentos1386/zdravko/internal/models"
|
"code.tjo.space/mentos1386/zdravko/internal/models"
|
||||||
"code.tjo.space/mentos1386/zdravko/internal/models/query"
|
"code.tjo.space/mentos1386/zdravko/internal/models/query"
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func CreateWorker(ctx context.Context, db *gorm.DB, worker *models.Worker) error {
|
func CreateWorker(ctx context.Context, q *query.Query, worker *models.Worker) error {
|
||||||
return db.WithContext(ctx).Create(worker).Error
|
return q.Worker.WithContext(ctx).Create(worker)
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetWorker(ctx context.Context, q *query.Query, slug string) (*models.Worker, error) {
|
func GetWorker(ctx context.Context, q *query.Query, slug string) (*models.Worker, error) {
|
||||||
|
|
|
@ -48,7 +48,7 @@ func (s *Server) Start() error {
|
||||||
}
|
}
|
||||||
log.Println("Connected to Temporal")
|
log.Println("Connected to Temporal")
|
||||||
|
|
||||||
h := handlers.NewBaseHandler(db, query, temporalClient, s.cfg)
|
h := handlers.NewBaseHandler(query, temporalClient, s.cfg)
|
||||||
|
|
||||||
// Health
|
// Health
|
||||||
s.echo.GET("/health", func(c echo.Context) error {
|
s.echo.GET("/health", func(c echo.Context) error {
|
||||||
|
@ -82,6 +82,7 @@ func (s *Server) Start() error {
|
||||||
settings.GET("/healthchecks/create", h.SettingsHealthchecksCreateGET)
|
settings.GET("/healthchecks/create", h.SettingsHealthchecksCreateGET)
|
||||||
settings.POST("/healthchecks/create", h.SettingsHealthchecksCreatePOST)
|
settings.POST("/healthchecks/create", h.SettingsHealthchecksCreatePOST)
|
||||||
settings.GET("/healthchecks/:slug", h.SettingsHealthchecksDescribeGET)
|
settings.GET("/healthchecks/:slug", h.SettingsHealthchecksDescribeGET)
|
||||||
|
settings.POST("/healthchecks/:slug", h.SettingsHealthchecksDescribePOST)
|
||||||
settings.GET("/workers", h.SettingsWorkersGET)
|
settings.GET("/workers", h.SettingsWorkersGET)
|
||||||
settings.GET("/workers/create", h.SettingsWorkersCreateGET)
|
settings.GET("/workers/create", h.SettingsWorkersCreateGET)
|
||||||
settings.POST("/workers/create", h.SettingsWorkersCreatePOST)
|
settings.POST("/workers/create", h.SettingsWorkersCreatePOST)
|
||||||
|
|
|
@ -912,6 +912,11 @@ video {
|
||||||
background-color: rgb(255 255 255 / var(--tw-bg-opacity));
|
background-color: rgb(255 255 255 / var(--tw-bg-opacity));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.bg-yellow-100 {
|
||||||
|
--tw-bg-opacity: 1;
|
||||||
|
background-color: rgb(254 249 195 / var(--tw-bg-opacity));
|
||||||
|
}
|
||||||
|
|
||||||
.p-2 {
|
.p-2 {
|
||||||
padding: 0.5rem;
|
padding: 0.5rem;
|
||||||
}
|
}
|
||||||
|
@ -1124,6 +1129,11 @@ video {
|
||||||
color: rgb(255 255 255 / var(--tw-text-opacity));
|
color: rgb(255 255 255 / var(--tw-text-opacity));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.text-yellow-800 {
|
||||||
|
--tw-text-opacity: 1;
|
||||||
|
color: rgb(133 77 14 / var(--tw-text-opacity));
|
||||||
|
}
|
||||||
|
|
||||||
.underline {
|
.underline {
|
||||||
text-decoration-line: underline;
|
text-decoration-line: underline;
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,13 +66,10 @@
|
||||||
</td>
|
</td>
|
||||||
<td class="px-6 py-4">
|
<td class="px-6 py-4">
|
||||||
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800">
|
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800">
|
||||||
SUCCESS
|
ACTIVE
|
||||||
</span>
|
</span>
|
||||||
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-red-100 text-red-800">
|
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-yellow-100 text-yellow-800">
|
||||||
FAILURE
|
PAUSED
|
||||||
</span>
|
|
||||||
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-gray-100 text-gray-800">
|
|
||||||
UNKNOWN
|
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="px-6 py-4">
|
<td class="px-6 py-4">
|
||||||
|
|
|
@ -61,13 +61,10 @@
|
||||||
</td>
|
</td>
|
||||||
<td class="px-6 py-4">
|
<td class="px-6 py-4">
|
||||||
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800">
|
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800">
|
||||||
SUCCESS
|
ONLINE
|
||||||
</span>
|
</span>
|
||||||
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-red-100 text-red-800">
|
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-red-100 text-red-800">
|
||||||
FAILURE
|
OFFLINE
|
||||||
</span>
|
|
||||||
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-gray-100 text-gray-800">
|
|
||||||
UNKNOWN
|
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="px-6 py-4">
|
<td class="px-6 py-4">
|
||||||
|
|
Loading…
Reference in a new issue