diff --git a/CHANGELOG.md b/CHANGELOG.md index fa46e80c..c061c3e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- River Pro sequence jobs show where they stand in their sequence on the job page. A pending job says whether it's up next, waiting on another job, or halted by a cancelled or discarded job, with a link to that job, and its timeline's wait step is tinted to match. A cancelled or discarded job that halted its sequence says which jobs wait behind it. A new sequence page lists every job in a sequence in run order, highlights the job that halted it with Retry and Delete in place, and marks any running or retrying job blocking it. Deleting a job that halted its sequence now warns how many jobs will run without it and offers Retry instead. [PR #740](https://github.com/riverqueue/riverui/pull/740). + ## [v0.22.0] - 2026-10-24 ### Added diff --git a/Makefile b/Makefile index dce6269c..77203094 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,9 @@ clean: # Find each Go submodule from the top-level directory. Go commands only # consider the current module, so they must run separately for every module. -submodules := $(shell find . -name 'go\.mod' -exec dirname {} \;) +# Skip nested checkouts (like agent worktrees under `.claude`) and dependencies, +# whose modules would otherwise be linted and tested as if they were this one's. +submodules := $(shell find . \( -name .claude -o -name node_modules \) -prune -o -name 'go\.mod' -exec dirname {} \;) .PHONY: dev dev: fake_assets diff --git a/riverproui/endpoints.go b/riverproui/endpoints.go index 00cf7f4a..c93f6d5a 100644 --- a/riverproui/endpoints.go +++ b/riverproui/endpoints.go @@ -119,6 +119,11 @@ func (e *endpoints[TTx]) Extensions(ctx context.Context) (map[string]bool, error return nil, err } + hasSequenceQueries, err := prohandler.HasSequenceQueries(ctx, execTx, schema) + if err != nil { + return nil, err + } + hasWorkflowV2Tables, err := prohandler.HasWorkflowV2Tables(ctx, execTx, schema) if err != nil { return nil, err @@ -128,6 +133,7 @@ func (e *endpoints[TTx]) Extensions(ctx context.Context) (map[string]bool, error "dead_letter_jobs": hasDeadLetterTable, "durable_periodic_jobs": hasPeriodicJobTable, "producer_queries": true, + "sequence_queries": hasSequenceQueries, "workflow_queries": hasWorkflowV2Tables, "has_producer_table": hasProducerTable, "has_sequence_table": hasSequenceTable, @@ -167,8 +173,11 @@ func (e *endpoints[TTx]) MountEndpoints(archetype *baseservice.Archetype, logger apiendpoint.Mount(mux, prohandler.NewDeadLetterAutocompleteEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewDeadLetterJobGetEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewDeadLetterJobRetryEndpoint(bundle), mountOpts), + apiendpoint.Mount(mux, prohandler.NewJobSequenceGetEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewPeriodicJobListEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewProducerListEndpoint(bundle), mountOpts), + apiendpoint.Mount(mux, prohandler.NewSequenceGetEndpoint(bundle), mountOpts), + apiendpoint.Mount(mux, prohandler.NewSequenceJobListEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewWorkflowCancelEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewWorkflowGetEndpoint(bundle), mountOpts), apiendpoint.Mount(mux, prohandler.NewWorkflowListEndpoint(bundle), mountOpts), diff --git a/riverproui/internal/prohandler/sequence_endpoints.go b/riverproui/internal/prohandler/sequence_endpoints.go new file mode 100644 index 00000000..ff6036f8 --- /dev/null +++ b/riverproui/internal/prohandler/sequence_endpoints.go @@ -0,0 +1,438 @@ +package prohandler + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "sync/atomic" + "time" + + "github.com/riverqueue/apiframe/apiendpoint" + "github.com/riverqueue/apiframe/apierror" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/rivershared/util/dbutil" + "github.com/riverqueue/river/rivertype" + + "riverqueue.com/riverpro" +) + +// SequenceIndexNames are the indexes sequence inspection reads through. Both +// come with River Pro's first migration, and the SQLite queries name them +// explicitly, so inspecting without them fails instead of scanning. +var SequenceIndexNames = []string{ //nolint:gochecknoglobals + "idx_river_job_non_pending_seq_key_id_desc", + "river_job_sequence_scheduling", +} + +// HasSequenceQueries reports whether the indexes that sequence inspection +// needs are installed. +func HasSequenceQueries(ctx context.Context, exec riverdriver.Executor, schema string) (bool, error) { + for _, index := range SequenceIndexNames { + exists, err := exec.IndexExists(ctx, &riverdriver.IndexExistsParams{Index: index, Schema: schema}) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + } + return true, nil +} + +// sequenceQueryTimeout bounds a single sequence request. Inspection cost grows +// with the cancelled or discarded jobs that allow continuation ahead of a +// sequence's next job, and a page that polls shouldn't hold a connection for +// long on an unusual sequence. +const sequenceQueryTimeout = 5 * time.Second + +// sequencePendingCountMax caps how many pending jobs are counted for a halted +// sequence. Beyond this, the UI only says there are more. +const sequencePendingCountMax = 1000 + +// sequenceIndexCheck remembers that the sequence indexes exist, so a page that +// polls a sequence endpoint doesn't query the catalog on every request. +// Indexes are only removed by down migrations, which also remove sequences. +type sequenceIndexCheck struct { + confirmed atomic.Bool +} + +func (c *sequenceIndexCheck) require(ctx context.Context, exec riverdriver.Executor, schema string) error { + if c.confirmed.Load() { + return nil + } + supported, err := HasSequenceQueries(ctx, exec, schema) + if err != nil { + return fmt.Errorf("error checking sequence indexes: %w", err) + } + if !supported { + return apierror.NewBadRequest("Sequence queries require River Pro's sequence indexes. Run the River Pro migrations.") + } + c.confirmed.Store(true) + return nil +} + +type sequenceJobResponse struct { + ID int64 `json:"id,string"` + Kind string `json:"kind"` + Queue string `json:"queue"` + ScheduledAt time.Time `json:"scheduled_at"` + State rivertype.JobState `json:"state"` +} + +func sequenceJobResponseFrom(job *riverpro.SequenceJobRef) *sequenceJobResponse { + if job == nil { + return nil + } + return &sequenceJobResponse{ + ID: job.ID, + Kind: job.Kind, + Queue: job.Queue, + ScheduledAt: job.ScheduledAt.UTC(), + State: job.State, + } +} + +type sequencePendingCountResponse struct { + Count int `json:"count"` + More bool `json:"more"` +} + +// sequenceInspectionResponse is a sequence's promotion state. It's shared by +// the job and sequence endpoints. +type sequenceInspectionResponse struct { + Blockers []*sequenceJobResponse `json:"blockers"` + HaltedBy *sequenceJobResponse `json:"halted_by"` + Head *sequenceJobResponse `json:"head"` + Key string `json:"key"` + + // PendingCount is the sequence's pending job count, capped at + // sequencePendingCountMax. It's null when it wasn't requested. + PendingCount *sequencePendingCountResponse `json:"pending_count"` +} + +func sequenceInspectionResponseFrom(inspection *riverpro.SequenceInspectResult) *sequenceInspectionResponse { + blockers := make([]*sequenceJobResponse, len(inspection.Blockers)) + for i, blocker := range inspection.Blockers { + blockers[i] = sequenceJobResponseFrom(blocker) + } + return &sequenceInspectionResponse{ + Blockers: blockers, + HaltedBy: sequenceJobResponseFrom(inspection.HaltedBy), + Head: sequenceJobResponseFrom(inspection.Head), + Key: inspection.Key, + } +} + +func sequenceCountPendingTx[TTx any](ctx context.Context, client *riverpro.Client[TTx], tx TTx, key string) (*sequencePendingCountResponse, error) { + result, err := client.SequenceCountPendingTx(ctx, tx, &riverpro.SequenceCountPendingParams{ + Key: key, + Max: sequencePendingCountMax, + }) + if err != nil { + return nil, fmt.Errorf("error counting pending sequence jobs: %w", err) + } + return &sequencePendingCountResponse{Count: result.Count, More: result.HasMore}, nil +} + +// +// jobSequenceGetEndpoint +// + +type jobSequenceGetEndpoint[TTx any] struct { + ProAPIBundle[TTx] + apiendpoint.Endpoint[jobSequenceGetRequest, jobSequenceGetResponse] + + indexCheck sequenceIndexCheck +} + +// NewJobSequenceGetEndpoint returns an endpoint explaining where a job stands +// in its sequence. +func NewJobSequenceGetEndpoint[TTx any](bundle ProAPIBundle[TTx]) *jobSequenceGetEndpoint[TTx] { + return &jobSequenceGetEndpoint[TTx]{ProAPIBundle: bundle} +} + +func (*jobSequenceGetEndpoint[TTx]) Meta() *apiendpoint.EndpointMeta { + return &apiendpoint.EndpointMeta{ + Pattern: "GET /api/pro/jobs/{job_id}/sequence", + StatusCode: http.StatusOK, + } +} + +type jobSequenceGetRequest struct { + JobID int64 `json:"-" validate:"required,min=1"` // from ExtractRaw +} + +func (req *jobSequenceGetRequest) ExtractRaw(r *http.Request) error { + id, err := strconv.ParseInt(r.PathValue("job_id"), 10, 64) + if err != nil { + return apierror.NewBadRequestf("Couldn't convert job ID to int64: %s.", err) + } + req.JobID = id + return nil +} + +type jobSequenceGetResponse struct { + // Halting is true when the job is cancelled or discarded, doesn't let its + // sequence continue, and pending jobs follow it. + Halting bool `json:"halting"` + JobID int64 `json:"job_id,string"` + JobState rivertype.JobState `json:"job_state"` + Sequence *sequenceInspectionResponse `json:"sequence"` + + // WaitReason is one of `halted`, `none`, `up_next`, + // `waiting_behind_pending`, or `waiting_on_active`. + WaitReason riverpro.SequencePendingReason `json:"wait_reason"` + WaitingOn *sequenceJobResponse `json:"waiting_on"` +} + +func (a *jobSequenceGetEndpoint[TTx]) Execute(ctx context.Context, req *jobSequenceGetRequest) (*jobSequenceGetResponse, error) { + ctx, cancel := context.WithTimeout(ctx, sequenceQueryTimeout) + defer cancel() + + if err := a.indexCheck.require(ctx, a.DB, a.Client.Schema()); err != nil { + return nil, err + } + + // Inspect and count in one transaction so a halting job's count describes + // the same snapshot as its inspection. + return dbutil.WithTxV(ctx, a.DB, func(ctx context.Context, execTx riverdriver.ExecutorTx) (*jobSequenceGetResponse, error) { + tx := a.Driver.UnwrapTx(execTx) + + inspection, err := a.Client.SequenceJobInspectTx(ctx, tx, req.JobID) + if err != nil { + if errors.Is(err, rivertype.ErrNotFound) { + return nil, apierror.NewNotFoundf("Job %d isn't in a sequence.", req.JobID) + } + return nil, fmt.Errorf("error inspecting job sequence: %w", err) + } + + sequence := sequenceInspectionResponseFrom(inspection.Sequence) + // A halting job's delete warning says how many jobs would run without + // it. Other jobs don't need the count. + if inspection.Halting { + sequence.PendingCount, err = sequenceCountPendingTx(ctx, a.Client, tx, inspection.Sequence.Key) + if err != nil { + return nil, err + } + } + + return &jobSequenceGetResponse{ + Halting: inspection.Halting, + JobID: inspection.Job.ID, + JobState: inspection.Job.State, + Sequence: sequence, + WaitReason: inspection.PendingReason, + WaitingOn: sequenceJobResponseFrom(inspection.WaitingOn), + }, nil + }) +} + +// +// sequenceGetEndpoint +// + +type sequenceGetEndpoint[TTx any] struct { + ProAPIBundle[TTx] + apiendpoint.Endpoint[sequenceGetRequest, sequenceInspectionResponse] + + indexCheck sequenceIndexCheck +} + +// NewSequenceGetEndpoint returns an endpoint reporting a sequence's promotion +// state by key. +func NewSequenceGetEndpoint[TTx any](bundle ProAPIBundle[TTx]) *sequenceGetEndpoint[TTx] { + return &sequenceGetEndpoint[TTx]{ProAPIBundle: bundle} +} + +func (*sequenceGetEndpoint[TTx]) Meta() *apiendpoint.EndpointMeta { + // Sequence keys are often base64, so the key goes in the query string where + // `/` and `+` survive proxies that decode paths. + return &apiendpoint.EndpointMeta{ + Pattern: "GET /api/pro/sequence", + StatusCode: http.StatusOK, + } +} + +type sequenceGetRequest struct { + Key string `json:"-" validate:"required"` // from ExtractRaw +} + +func (req *sequenceGetRequest) ExtractRaw(r *http.Request) error { + req.Key = r.URL.Query().Get("key") + return nil +} + +func (a *sequenceGetEndpoint[TTx]) Execute(ctx context.Context, req *sequenceGetRequest) (*sequenceInspectionResponse, error) { + ctx, cancel := context.WithTimeout(ctx, sequenceQueryTimeout) + defer cancel() + + if err := a.indexCheck.require(ctx, a.DB, a.Client.Schema()); err != nil { + return nil, err + } + + return dbutil.WithTxV(ctx, a.DB, func(ctx context.Context, execTx riverdriver.ExecutorTx) (*sequenceInspectionResponse, error) { + tx := a.Driver.UnwrapTx(execTx) + + inspection, err := a.Client.SequenceInspectTx(ctx, tx, req.Key) + if err != nil { + return nil, fmt.Errorf("error inspecting sequence: %w", err) + } + + resp := sequenceInspectionResponseFrom(inspection) + resp.PendingCount = &sequencePendingCountResponse{} + // Nothing is pending without a head, so skip the count. + if inspection.Head != nil { + resp.PendingCount, err = sequenceCountPendingTx(ctx, a.Client, tx, inspection.Key) + if err != nil { + return nil, err + } + } + return resp, nil + }) +} + +// +// sequenceJobListEndpoint +// + +type sequenceJobListEndpoint[TTx any] struct { + ProAPIBundle[TTx] + apiendpoint.Endpoint[sequenceJobListRequest, sequenceJobListResponse] + + indexCheck sequenceIndexCheck +} + +// NewSequenceJobListEndpoint returns an endpoint listing a page of a +// sequence's jobs in ID order. +func NewSequenceJobListEndpoint[TTx any](bundle ProAPIBundle[TTx]) *sequenceJobListEndpoint[TTx] { + return &sequenceJobListEndpoint[TTx]{ProAPIBundle: bundle} +} + +func (*sequenceJobListEndpoint[TTx]) Meta() *apiendpoint.EndpointMeta { + return &apiendpoint.EndpointMeta{ + Pattern: "GET /api/pro/sequence/jobs", + StatusCode: http.StatusOK, + } +} + +type sequenceJobListRequest struct { + AfterID *int64 `json:"-" validate:"omitempty,min=0"` // from ExtractRaw + BeforeID *int64 `json:"-" validate:"omitempty,min=0"` // from ExtractRaw + Key string `json:"-" validate:"required"` // from ExtractRaw + Limit int `json:"-" validate:"omitempty,min=1,max=1000"` // from ExtractRaw +} + +func (req *sequenceJobListRequest) ExtractRaw(r *http.Request) error { + query := r.URL.Query() + req.Key = query.Get("key") + + // A cursor of zero is meaningful: `after_id=0` lists from the first job. + parseID := func(name string) (*int64, error) { + value := query.Get(name) + if value == "" { + return nil, nil //nolint:nilnil + } + id, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, apierror.NewBadRequestf("Couldn't convert `%s` to int64: %s.", name, err) + } + return &id, nil + } + + var err error + if req.AfterID, err = parseID("after_id"); err != nil { + return err + } + if req.BeforeID, err = parseID("before_id"); err != nil { + return err + } + if req.AfterID != nil && req.BeforeID != nil { + return apierror.NewBadRequest("Only one of `after_id` and `before_id` may be set.") + } + + if limitStr := query.Get("limit"); limitStr != "" { + limit, err := strconv.Atoi(limitStr) + if err != nil { + return apierror.NewBadRequestf("Couldn't convert `limit` to integer: %s.", err) + } + req.Limit = limit + } + return nil +} + +type sequenceJobListItemResponse struct { + ID int64 `json:"id,string"` + Args string `json:"args"` + Attempt int `json:"attempt"` + AttemptedAt *time.Time `json:"attempted_at"` + CreatedAt time.Time `json:"created_at"` + FinalizedAt *time.Time `json:"finalized_at"` + Kind string `json:"kind"` + MaxAttempts int `json:"max_attempts"` + Priority int `json:"priority"` + Queue string `json:"queue"` + ScheduledAt time.Time `json:"scheduled_at"` + State rivertype.JobState `json:"state"` +} + +// sequenceJobListResponse is a page of a sequence's jobs in ascending ID +// order, which is run order. HasEarlier and HasLater say whether jobs exist +// past either end of the page, so the UI can offer to load them. +type sequenceJobListResponse struct { + Data []*sequenceJobListItemResponse `json:"data"` + HasEarlier bool `json:"has_earlier"` + HasLater bool `json:"has_later"` +} + +func (a *sequenceJobListEndpoint[TTx]) Execute(ctx context.Context, req *sequenceJobListRequest) (*sequenceJobListResponse, error) { + ctx, cancel := context.WithTimeout(ctx, sequenceQueryTimeout) + defer cancel() + + if err := a.indexCheck.require(ctx, a.DB, a.Client.Schema()); err != nil { + return nil, err + } + + return dbutil.WithTxV(ctx, a.DB, func(ctx context.Context, execTx riverdriver.ExecutorTx) (*sequenceJobListResponse, error) { + tx := a.Driver.UnwrapTx(execTx) + + page, err := a.Client.SequenceJobListTx(ctx, tx, &riverpro.SequenceJobListParams{ + AfterID: req.AfterID, + BeforeID: req.BeforeID, + Key: req.Key, + Max: req.Limit, + }) + if err != nil { + return nil, fmt.Errorf("error listing sequence jobs: %w", err) + } + + resp := &sequenceJobListResponse{ + Data: make([]*sequenceJobListItemResponse, len(page.Jobs)), + HasEarlier: page.HasEarlier, + HasLater: page.HasLater, + } + for i, job := range page.Jobs { + resp.Data[i] = sequenceJobListItemResponseFrom(job) + } + return resp, nil + }) +} + +func sequenceJobListItemResponseFrom(item *riverpro.SequenceJobListItem) *sequenceJobListItemResponse { + job := item.Job + return &sequenceJobListItemResponse{ + ID: job.ID, + Args: string(job.EncodedArgs), + Attempt: job.Attempt, + AttemptedAt: job.AttemptedAt, + CreatedAt: job.CreatedAt, + FinalizedAt: job.FinalizedAt, + Kind: job.Kind, + MaxAttempts: job.MaxAttempts, + Priority: job.Priority, + Queue: job.Queue, + ScheduledAt: job.ScheduledAt.UTC(), + State: job.State, + } +} diff --git a/riverproui/internal/prohandler/sequence_endpoints_test.go b/riverproui/internal/prohandler/sequence_endpoints_test.go new file mode 100644 index 00000000..dd0b78ba --- /dev/null +++ b/riverproui/internal/prohandler/sequence_endpoints_test.go @@ -0,0 +1,357 @@ +package prohandler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/riverqueue/apiframe/apierror" + "github.com/riverqueue/apiframe/apitest" + "github.com/riverqueue/river/rivershared/util/dbutil" + "github.com/riverqueue/river/rivertype" + + "riverqueue.com/riverui/internal/riverinternaltest/testfactory" +) + +// insertSequenceJobs inserts one job per state into a sequence, in order, and +// returns their IDs. +func insertSequenceJobs(ctx context.Context, t *testing.T, bundle *setupEndpointTestBundle, key string, states ...rivertype.JobState) []int64 { + t.Helper() + + metadata, err := json.Marshal(map[string]string{"seq_key": key}) + require.NoError(t, err) + + ids := make([]int64, len(states)) + for i, state := range states { + opts := &testfactory.JobOpts{Metadata: metadata, State: &state} + switch state { //nolint:exhaustive + case rivertype.JobStateCancelled, rivertype.JobStateCompleted, rivertype.JobStateDiscarded: + opts.FinalizedAt = new(time.Now()) + } + params := testfactory.Job_Build(t, opts) + params.Schema = bundle.schema + job, err := bundle.exec.JobInsertFull(ctx, params) + require.NoError(t, err) + ids[i] = job.ID + } + return ids +} + +func TestJobSequenceGetEndpoint(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("Halted", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + ids := insertSequenceJobs(ctx, t, bundle, "order", + rivertype.JobStateCompleted, rivertype.JobStateCancelled, rivertype.JobStatePending, rivertype.JobStatePending) + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: ids[3]}) + require.NoError(t, err) + require.False(t, resp.Halting) + require.Equal(t, ids[3], resp.JobID) + require.Equal(t, rivertype.JobStatePending, resp.JobState) + require.Equal(t, "halted", string(resp.WaitReason)) + require.Equal(t, ids[1], resp.WaitingOn.ID) + require.Equal(t, "order", resp.Sequence.Key) + require.Equal(t, ids[1], resp.Sequence.HaltedBy.ID) + require.Equal(t, ids[2], resp.Sequence.Head.ID) + // Only a halting job's response counts the jobs it holds back. + require.Nil(t, resp.Sequence.PendingCount) + + resp, err = apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: ids[1]}) + require.NoError(t, err) + require.True(t, resp.Halting) + require.Equal(t, "none", string(resp.WaitReason)) + require.Nil(t, resp.WaitingOn) + require.Equal(t, &sequencePendingCountResponse{Count: 2}, resp.Sequence.PendingCount) + }) + + t.Run("WaitingOnActive", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + ids := insertSequenceJobs(ctx, t, bundle, "order", + rivertype.JobStateRunning, rivertype.JobStatePending, rivertype.JobStatePending) + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: ids[1]}) + require.NoError(t, err) + require.Equal(t, "waiting_on_active", string(resp.WaitReason)) + require.Equal(t, ids[0], resp.WaitingOn.ID) + require.Equal(t, rivertype.JobStateRunning, resp.WaitingOn.State) + + resp, err = apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: ids[2]}) + require.NoError(t, err) + require.Equal(t, "waiting_behind_pending", string(resp.WaitReason)) + require.Equal(t, ids[1], resp.WaitingOn.ID) + }) + + t.Run("UpNext", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + ids := insertSequenceJobs(ctx, t, bundle, "order", rivertype.JobStateCompleted, rivertype.JobStatePending) + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: ids[1]}) + require.NoError(t, err) + require.Equal(t, "up_next", string(resp.WaitReason)) + require.Nil(t, resp.WaitingOn) + }) + + t.Run("LargeIDsEncodeAsStrings", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + ids := insertSequenceJobs(ctx, t, bundle, "order", rivertype.JobStateCancelled, rivertype.JobStatePending) + // IDs past 2^53 lose precision as JSON numbers in the browser. + const largeID int64 = 1970670598291982290 + for i, id := range ids { + require.NoError(t, bundle.exec.Exec(ctx, "UPDATE "+dbutil.SafeIdentifier(bundle.schema)+".river_job SET id = $1 WHERE id = $2", largeID+int64(i), id)) + } + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: largeID + 1}) + require.NoError(t, err) + encoded, err := json.Marshal(resp) + require.NoError(t, err) + require.Contains(t, string(encoded), `"job_id":"1970670598291982291"`) + require.Contains(t, string(encoded), `"id":"1970670598291982290"`) + }) + + t.Run("NotInSequence", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + params := testfactory.Job_Build(t, nil) + params.Schema = bundle.schema + job, err := bundle.exec.JobInsertFull(ctx, params) + require.NoError(t, err) + + for _, id := range []int64{job.ID, 9223372036854775807} { + _, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: id}) + require.Equal(t, apierror.NewNotFoundf("Job %d isn't in a sequence.", id), err) + } + }) + + t.Run("MissingIndex", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + require.NoError(t, bundle.exec.Exec(ctx, "DROP INDEX "+dbutil.SafeIdentifier(bundle.schema)+".river_job_sequence_scheduling")) + + _, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: 1}) + require.ErrorContains(t, err, "Sequence queries require River Pro's sequence indexes") + }) + + t.Run("IndexCheckCached", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + req := &jobSequenceGetRequest{JobID: 9223372036854775807} + _, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), req) + require.ErrorContains(t, err, "isn't in a sequence") + + // A polling page skips the catalog check after the indexes are seen. + require.NoError(t, bundle.exec.Exec(ctx, "DROP INDEX "+dbutil.SafeIdentifier(bundle.schema)+".river_job_sequence_scheduling")) + _, err = apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), req) + require.ErrorContains(t, err, "isn't in a sequence") + }) + + t.Run("Validation", func(t *testing.T) { + t.Parallel() + + for _, value := range []string{"not-a-number", "9223372036854775808"} { + httpReq := httptest.NewRequestWithContext(ctx, http.MethodGet, "/", nil) + httpReq.SetPathValue("job_id", value) + require.Error(t, new(jobSequenceGetRequest).ExtractRaw(httpReq)) + } + + endpoint, _ := setupEndpoint(ctx, t, NewJobSequenceGetEndpoint) + for _, id := range []int64{0, -1} { + _, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &jobSequenceGetRequest{JobID: id}) + require.Error(t, err) + } + }) +} + +func TestSequenceGetEndpoint(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("Halted", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewSequenceGetEndpoint) + ids := insertSequenceJobs(ctx, t, bundle, "a/b+c=", + rivertype.JobStateCompleted, rivertype.JobStateDiscarded, rivertype.JobStatePending, rivertype.JobStatePending, rivertype.JobStatePending) + _ = insertSequenceJobs(ctx, t, bundle, "other", rivertype.JobStatePending) + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &sequenceGetRequest{Key: "a/b+c="}) + require.NoError(t, err) + require.Equal(t, "a/b+c=", resp.Key) + require.Equal(t, ids[1], resp.HaltedBy.ID) + require.Equal(t, rivertype.JobStateDiscarded, resp.HaltedBy.State) + require.Equal(t, ids[2], resp.Head.ID) + require.Len(t, resp.Blockers, 1) + require.Equal(t, &sequencePendingCountResponse{Count: 3}, resp.PendingCount) + }) + + t.Run("UnknownKey", func(t *testing.T) { + t.Parallel() + + endpoint, _ := setupEndpoint(ctx, t, NewSequenceGetEndpoint) + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &sequenceGetRequest{Key: "missing"}) + require.NoError(t, err) + require.Equal(t, &sequenceInspectionResponse{ + Blockers: []*sequenceJobResponse{}, + Key: "missing", + PendingCount: &sequencePendingCountResponse{}, + }, resp) + }) + + t.Run("KeyRequired", func(t *testing.T) { + t.Parallel() + + endpoint, _ := setupEndpoint(ctx, t, NewSequenceGetEndpoint) + + _, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &sequenceGetRequest{}) + require.Error(t, err) + }) +} + +func TestSequenceJobListEndpoint(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + listIDs := func(resp *sequenceJobListResponse) []int64 { + ids := make([]int64, len(resp.Data)) + for i, job := range resp.Data { + ids[i] = job.ID + } + return ids + } + + t.Run("Paging", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewSequenceJobListEndpoint) + ids := insertSequenceJobs(ctx, t, bundle, "order", + rivertype.JobStateCompleted, rivertype.JobStateCompleted, rivertype.JobStateCancelled, rivertype.JobStatePending, rivertype.JobStatePending) + _ = insertSequenceJobs(ctx, t, bundle, "other", rivertype.JobStatePending) + + invoke := func(req *sequenceJobListRequest) *sequenceJobListResponse { + t.Helper() + req.Key = "order" + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), req) + require.NoError(t, err) + return resp + } + + // The default page holds the newest jobs. + resp := invoke(&sequenceJobListRequest{Limit: 2}) + require.Equal(t, ids[3:], listIDs(resp)) + require.True(t, resp.HasEarlier) + require.False(t, resp.HasLater) + + // Forward from before the halting job, with jobs on both sides. + resp = invoke(&sequenceJobListRequest{AfterID: new(ids[1]), Limit: 2}) + require.Equal(t, ids[2:4], listIDs(resp)) + require.True(t, resp.HasEarlier) + require.True(t, resp.HasLater) + + // An `after_id` of zero lists from the first job. + resp = invoke(&sequenceJobListRequest{AfterID: new(int64(0)), Limit: 2}) + require.Equal(t, ids[:2], listIDs(resp)) + require.False(t, resp.HasEarlier) + require.True(t, resp.HasLater) + + // Forward from the start. + resp = invoke(&sequenceJobListRequest{AfterID: new(ids[0] - 1), Limit: 10}) + require.Equal(t, ids, listIDs(resp)) + require.False(t, resp.HasEarlier) + require.False(t, resp.HasLater) + + // Backward toward the oldest job. + resp = invoke(&sequenceJobListRequest{BeforeID: new(ids[2]), Limit: 1}) + require.Equal(t, ids[1:2], listIDs(resp)) + require.True(t, resp.HasEarlier) + require.True(t, resp.HasLater) + + resp = invoke(&sequenceJobListRequest{BeforeID: new(ids[2]), Limit: 5}) + require.Equal(t, ids[:2], listIDs(resp)) + require.False(t, resp.HasEarlier) + require.True(t, resp.HasLater) + + // Empty pages past either end still say what's on the other side. + resp = invoke(&sequenceJobListRequest{AfterID: new(ids[4])}) + require.Empty(t, resp.Data) + require.True(t, resp.HasEarlier) + require.False(t, resp.HasLater) + + resp = invoke(&sequenceJobListRequest{BeforeID: new(ids[0])}) + require.Empty(t, resp.Data) + require.False(t, resp.HasEarlier) + require.True(t, resp.HasLater) + }) + + t.Run("Fields", func(t *testing.T) { + t.Parallel() + + endpoint, bundle := setupEndpoint(ctx, t, NewSequenceJobListEndpoint) + _ = insertSequenceJobs(ctx, t, bundle, "order", rivertype.JobStateCompleted) + + resp, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), &sequenceJobListRequest{Key: "order"}) + require.NoError(t, err) + require.Len(t, resp.Data, 1) + job := resp.Data[0] + require.JSONEq(t, `{}`, job.Args) + require.Equal(t, rivertype.JobStateCompleted, job.State) + require.NotNil(t, job.FinalizedAt) + require.NotEmpty(t, job.Kind) + }) + + t.Run("ExtractRaw", func(t *testing.T) { + t.Parallel() + + httpReq := httptest.NewRequestWithContext(ctx, http.MethodGet, "/api/pro/sequence/jobs?key=a%2Fb%2Bc%3D&after_id=42&limit=200", nil) + req := new(sequenceJobListRequest) + require.NoError(t, req.ExtractRaw(httpReq)) + require.Equal(t, &sequenceJobListRequest{AfterID: new(int64(42)), Key: "a/b+c=", Limit: 200}, req) + + for _, url := range []string{ + "/?key=k&after_id=1&before_id=2", + "/?key=k&after_id=nope", + "/?key=k&before_id=nope", + "/?key=k&limit=nope", + } { + httpReq := httptest.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.Error(t, new(sequenceJobListRequest).ExtractRaw(httpReq), url) + } + }) + + t.Run("Validation", func(t *testing.T) { + t.Parallel() + + endpoint, _ := setupEndpoint(ctx, t, NewSequenceJobListEndpoint) + for _, req := range []*sequenceJobListRequest{ + {}, + {Key: "k", AfterID: new(int64(-1))}, + {Key: "k", BeforeID: new(int64(-1))}, + {Key: "k", Limit: 1001}, + {Key: "k", Limit: -1}, + } { + _, err := apitest.InvokeHandler(ctx, endpoint.Execute, testMountOpts(t), req) + require.Error(t, err) + } + }) +} diff --git a/riverproui/pro_handler_test.go b/riverproui/pro_handler_test.go index 9adf41d0..e611143d 100644 --- a/riverproui/pro_handler_test.go +++ b/riverproui/pro_handler_test.go @@ -9,6 +9,7 @@ import ( "log/slog" "net/http" "net/http/httptest" + "net/url" "testing" "time" @@ -182,6 +183,15 @@ func runProHandlerRequests(t *testing.T, schema string, exec riverdriver.Executo makeAPICall(t, "FeaturesGet", http.MethodGet, "/api/features", nil) makeAPICall(t, "PeriodicJobList", http.MethodGet, "/api/pro/periodic-jobs", nil) + + sequenceKey := "a/b+c=" + sequenceMetadata := uicommontest.MustMarshalJSON(t, map[string]string{"seq_key": sequenceKey}) + _ = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: new(time.Now()), Metadata: sequenceMetadata, State: new(rivertype.JobStateCancelled)}) + sequenceJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Metadata: sequenceMetadata, State: new(rivertype.JobStatePending)}) + makeAPICall(t, "JobSequenceGet", http.MethodGet, fmt.Sprintf("/api/pro/jobs/%d/sequence", sequenceJob.ID), nil) + makeAPICall(t, "SequenceGet", http.MethodGet, "/api/pro/sequence?key="+url.QueryEscape(sequenceKey), nil) + makeAPICall(t, "SequenceJobList", http.MethodGet, "/api/pro/sequence/jobs?key="+url.QueryEscape(sequenceKey), nil) + makeAPICall(t, "SequenceJobListAfter", http.MethodGet, fmt.Sprintf("/api/pro/sequence/jobs?key=%s&after_id=%d&limit=1", url.QueryEscape(sequenceKey), sequenceJob.ID-1), nil) makeAPICall(t, "ProducerList", http.MethodGet, "/api/pro/producers?queue_name="+queue.Name, nil) makeAPICall(t, "WorkflowCancel", http.MethodPost, fmt.Sprintf("/api/pro/workflows/%s/cancel", workflowID), nil) makeAPICall(t, "WorkflowGet", http.MethodGet, fmt.Sprintf("/api/pro/workflows/%s", workflowID2), nil) @@ -264,10 +274,30 @@ func TestProMountedEndpointResponses(t *testing.T) { "has_producer_table": true, // dynamic "has_sequence_table": true, // dynamic "producer_queries": true, // static + "sequence_queries": true, // dynamic "workflow_queries": true, // dynamic } require.Equal(t, expectedExtensions, resp.Extensions) + recorder = httptest.NewRecorder() + req = httptest.NewRequestWithContext(ctx, http.MethodGet, "/api/pro/jobs/9223372036854775807/sequence", nil) + bundle.handler.ServeHTTP(recorder, req) + require.Equal(t, http.StatusNotFound, recorder.Result().StatusCode) + require.Contains(t, recorder.Body.String(), "isn't in a sequence") + + // Keys are commonly base64, so they're passed in the query string. + recorder = httptest.NewRecorder() + req = httptest.NewRequestWithContext(ctx, http.MethodGet, "/api/pro/sequence?key="+url.QueryEscape("a/b+c="), nil) + bundle.handler.ServeHTTP(recorder, req) + require.Equal(t, http.StatusOK, recorder.Result().StatusCode) + require.JSONEq(t, `{"blockers":[],"halted_by":null,"head":null,"key":"a/b+c=","pending_count":{"count":0,"more":false}}`, recorder.Body.String()) + + recorder = httptest.NewRecorder() + req = httptest.NewRequestWithContext(ctx, http.MethodGet, "/api/pro/sequence/jobs?key="+url.QueryEscape("a/b+c="), nil) + bundle.handler.ServeHTTP(recorder, req) + require.Equal(t, http.StatusOK, recorder.Result().StatusCode) + require.JSONEq(t, `{"data":[],"has_earlier":false,"has_later":false}`, recorder.Body.String()) + recorder = httptest.NewRecorder() req = httptest.NewRequestWithContext( ctx, diff --git a/src/components/CopyableHeadingText.tsx b/src/components/CopyableHeadingText.tsx index abb0dc44..5c65e116 100644 --- a/src/components/CopyableHeadingText.tsx +++ b/src/components/CopyableHeadingText.tsx @@ -2,12 +2,15 @@ import { CheckIcon } from "@heroicons/react/16/solid"; import { ClipboardIcon } from "@heroicons/react/24/outline"; import { toastSuccess } from "@services/toast"; import { capitalize } from "@utils/string"; +import clsx from "clsx"; import { useState } from "react"; type CopyableHeadingTextProps = { // Names what is copied, in lowercase, such as "job kind". label: string; text: string; + // Replaces the default heading weight and color, for use in a value cell. + textClassName?: string; }; // CopyableHeadingText keeps a heading on one line. Text that doesn't fit @@ -16,6 +19,7 @@ type CopyableHeadingTextProps = { export default function CopyableHeadingText({ label, text, + textClassName = "font-semibold text-slate-900 dark:text-white", }: CopyableHeadingTextProps) { const [copied, setCopied] = useState(false); @@ -40,7 +44,7 @@ export default function CopyableHeadingText({ return ( {text} diff --git a/src/components/DeleteJobConfirmation.test.tsx b/src/components/DeleteJobConfirmation.test.tsx new file mode 100644 index 00000000..0cec1699 --- /dev/null +++ b/src/components/DeleteJobConfirmation.test.tsx @@ -0,0 +1,104 @@ +import type { JobSequenceStatus } from "@services/sequences"; + +import { JobState } from "@services/types"; +import { + jobSequence, + sequenceInspection, + sequenceJob, + sequenceKeyFixture, +} from "@test/factories/sequence"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +import DeleteJobConfirmation from "./DeleteJobConfirmation"; + +const halting = sequenceJob(7001n, JobState.Cancelled); +const head = sequenceJob(7003n, JobState.Pending); + +const haltingStatus = ( + overrides: Partial = {}, + haltedBy = halting, +): JobSequenceStatus => ({ + error: false, + jobSequence: jobSequence({ + halting: true, + jobId: 7001n, + jobState: JobState.Cancelled, + sequence: sequenceInspection({ + haltedBy, + head, + pendingCount: { count: 2341, more: false }, + }), + }), + key: sequenceKeyFixture, + ...overrides, +}); + +const renderDialog = (sequence?: JobSequenceStatus) => { + const handlers = { onClose: vi.fn(), onDelete: vi.fn(), onRetry: vi.fn() }; + render( + , + ); + return handlers; +}; + +describe("DeleteJobConfirmation", () => { + test("uses the plain confirmation for a job outside a sequence", () => { + const { onDelete } = renderDialog(); + expect(screen.getByText("Delete job?")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Delete job" })); + expect(onDelete).toHaveBeenCalledTimes(1); + }); + + test("uses the plain confirmation for a job that isn't halting", () => { + renderDialog({ + error: false, + jobSequence: jobSequence({ jobState: JobState.Completed }), + key: sequenceKeyFixture, + }); + expect(screen.getByText("Delete job?")).toBeInTheDocument(); + }); + + test("uses the plain confirmation when the sequence failed to load", () => { + renderDialog(haltingStatus({ error: true })); + expect(screen.getByText("Delete job?")).toBeInTheDocument(); + }); + + test("says how many jobs a halting job holds back and offers retry", () => { + const { onDelete, onRetry } = renderDialog(haltingStatus()); + const dialog = screen.getByRole("dialog"); + expect(dialog).toHaveTextContent("Job 7001 halted its sequence"); + expect(dialog).toHaveTextContent( + "2,341 pending jobs are waiting behind it, starting with #7003.", + ); + expect(dialog).toHaveTextContent( + "The 2,341 pending jobs start running right away, skipping it.", + ); + + fireEvent.click(screen.getByRole("button", { name: "Retry instead" })); + expect(onRetry).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByRole("button", { name: "Delete anyway" })); + expect(onDelete).toHaveBeenCalledTimes(1); + }); + + test("focuses retry so Enter doesn't delete", async () => { + renderDialog(haltingStatus()); + await waitFor(() => { + expect( + screen.getByRole("button", { name: "Retry instead" }), + ).toHaveFocus(); + }); + }); + + test("explains that an earlier halting job still holds the sequence", () => { + renderDialog(haltingStatus({}, sequenceJob(6998n, JobState.Discarded))); + expect(screen.getByRole("dialog")).toHaveTextContent( + "The jobs behind it keep waiting on earlier job #6998, then run without this one.", + ); + }); +}); diff --git a/src/components/DeleteJobConfirmation.tsx b/src/components/DeleteJobConfirmation.tsx new file mode 100644 index 00000000..552d2dac --- /dev/null +++ b/src/components/DeleteJobConfirmation.tsx @@ -0,0 +1,58 @@ +import ConfirmationDialog from "@components/ConfirmationDialog"; +import HaltingJobDeleteDialog from "@components/HaltingJobDeleteDialog"; +import { type JobSequenceStatus } from "@services/sequences"; + +// DeleteJobConfirmation confirms deleting a job. A job that's halting its +// sequence gets a dialog explaining that the jobs behind it will run without +// it, and offering Retry instead. +export default function DeleteJobConfirmation({ + job, + onClose, + onDelete, + onRetry, + open, + sequence, +}: { + job: { id: bigint }; + onClose: () => void; + onDelete: () => void; + onRetry: () => void; + open: boolean; + sequence?: JobSequenceStatus; +}) { + const jobSequence = sequence?.error ? undefined : sequence?.jobSequence; + const head = jobSequence?.sequence.head; + const pendingCount = jobSequence?.sequence.pendingCount; + if (jobSequence?.halting && head && pendingCount) { + const { haltedBy } = jobSequence.sequence; + return ( + + ); + } + + return ( + + This permanently deletes job{" "} + {job.id.toString()}. This action + cannot be undone. + + } + onClose={onClose} + onConfirm={onDelete} + open={open} + title="Delete job?" + /> + ); +} diff --git a/src/components/HaltingJobDeleteDialog.stories.tsx b/src/components/HaltingJobDeleteDialog.stories.tsx new file mode 100644 index 00000000..33c650e5 --- /dev/null +++ b/src/components/HaltingJobDeleteDialog.stories.tsx @@ -0,0 +1,45 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; + +import { JobState } from "@services/types"; +import { sequenceJob } from "@test/factories/sequence"; +import { fn } from "storybook/test"; + +import HaltingJobDeleteDialog from "./HaltingJobDeleteDialog"; + +const meta: Meta = { + args: { + head: sequenceJob(7003n, JobState.Pending), + jobId: 7001n, + onClose: fn(), + onDelete: fn(), + onRetry: fn(), + open: true, + pendingCount: { count: 2341, more: false }, + }, + component: HaltingJobDeleteDialog, + title: "Components/HaltingJobDeleteDialog", +}; + +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; + +export const OnePendingJob: Story = { + args: { pendingCount: { count: 1, more: false } }, +}; + +// Counts are capped, so a long backlog reads "1,000+". +export const CappedCount: Story = { + args: { pendingCount: { count: 1000, more: true } }, +}; + +// When an earlier job also halts the sequence, deleting this one doesn't +// release the waiting jobs yet. +export const EarlierHalt: Story = { + args: { + earlierHalt: sequenceJob(6998n, JobState.Discarded), + jobId: 7001n, + }, +}; diff --git a/src/components/HaltingJobDeleteDialog.tsx b/src/components/HaltingJobDeleteDialog.tsx new file mode 100644 index 00000000..d3948bab --- /dev/null +++ b/src/components/HaltingJobDeleteDialog.tsx @@ -0,0 +1,193 @@ +import { + Description, + Dialog, + DialogBackdrop, + DialogPanel, + DialogTitle, +} from "@headlessui/react"; +import { + ArrowUturnLeftIcon, + ExclamationTriangleIcon, + TrashIcon, +} from "@heroicons/react/24/outline"; +import { + formatPendingCount, + type SequenceJob, + type SequencePendingCount, +} from "@services/sequences"; +import { type ReactNode } from "react"; + +export type HaltingJobDeleteDialogProps = { + // An earlier job that also halts the sequence, if any. Deleting this job + // doesn't release the waiting jobs while it's there. + earlierHalt?: null | SequenceJob; + // The first pending job behind the halting job. + head: SequenceJob; + jobId: bigint; + onClose: () => void; + onDelete: () => void; + onRetry: () => void; + open: boolean; + pendingCount: SequencePendingCount; +}; + +const pendingJobsPhrase = (count: SequencePendingCount) => + `${formatPendingCount(count)} pending ${count.count === 1 && !count.more ? "job" : "jobs"}`; + +// HaltingJobDeleteDialog confirms deleting a cancelled or discarded job that's +// halting its sequence. Deleting it lets the jobs behind it run without it, +// which is rarely what an operator wants, so the dialog lays out both choices +// side by side and offers Retry, which keeps the sequence in order. +export default function HaltingJobDeleteDialog({ + earlierHalt, + head, + jobId, + onClose, + onDelete, + onRetry, + open, + pendingCount, +}: HaltingJobDeleteDialogProps) { + if (!open) return null; + + const id = jobId.toString(); + const waiting = pendingJobsPhrase(pendingCount); + + return ( + + + +
+
+ +
+
+
+
+
+ + Job {id} halted its + sequence + + + + {waiting} + {" "} + are waiting behind it, starting with{" "} + #{head.id.toString()}. + +
+
+ +
+ + + + +
+
+
+ +
+
+
+
+
+ ); +} + +function Choice({ + action, + badge, + badgeClassName, + children, + title, +}: { + action: ReactNode; + badge: string; + badgeClassName: string; + children: ReactNode; + title: string; +}) { + return ( +
+
+

+ {title} +

+ + {badge} + +
+

+ {children} +

+ {action} +
+ ); +} diff --git a/src/components/JobDetail.stories.ts b/src/components/JobDetail.stories.ts index c14e5677..f095ca54 100644 --- a/src/components/JobDetail.stories.ts +++ b/src/components/JobDetail.stories.ts @@ -1,6 +1,15 @@ +import type { Job } from "@services/jobs"; +import type { JobSequence, JobSequenceStatus } from "@services/sequences"; import type { Meta, StoryObj } from "@storybook/react-vite"; +import { JobState } from "@services/types"; import { jobFactory } from "@test/factories/job"; +import { + jobSequence, + sequenceInspection, + sequenceJob, + sequenceKeyFixture, +} from "@test/factories/sequence"; import JobDetail from "./JobDetail"; @@ -54,3 +63,139 @@ export const Cancelled: Story = { job: jobFactory.cancelled().build(), }, }; + +// Jobs in a sequence get a Sequence field and, while pending, a wait reason +// on the timeline. IDs follow one order's sequence: 7001 was cancelled and +// halts it, and 7003 is the next pending job. +const inSequence = (job: Job): Job => ({ + ...job, + metadata: { ...job.metadata, seq_key: sequenceKeyFixture }, +}); + +const sequenceStatus = ( + overrides: Partial = {}, +): JobSequenceStatus => ({ + error: false, + jobSequence: jobSequence(overrides), + key: sequenceKeyFixture, +}); + +const pendingInSequence = inSequence(jobFactory.pending().build({ id: 7003n })); +const halting = sequenceJob(7001n, JobState.Cancelled); +const head = sequenceJob(7003n, JobState.Pending); + +export const SequenceHalted: Story = { + args: { + job: pendingInSequence, + sequence: sequenceStatus({ + sequence: sequenceInspection({ + blockers: [halting], + haltedBy: halting, + head, + }), + waitingOn: halting, + waitReason: "halted", + }), + }, +}; + +export const SequenceWaitingOnRunningJob: Story = { + args: { + job: pendingInSequence, + sequence: sequenceStatus({ + waitingOn: sequenceJob(7002n, JobState.Running), + waitReason: "waiting_on_active", + }), + }, +}; + +export const SequenceWaitingOnRetryableJob: Story = { + args: { + job: pendingInSequence, + sequence: sequenceStatus({ + waitingOn: sequenceJob(7002n, JobState.Retryable), + waitReason: "waiting_on_active", + }), + }, +}; + +export const SequenceWaitingBehindPendingJob: Story = { + args: { + job: inSequence(jobFactory.pending().build({ id: 7010n })), + sequence: sequenceStatus({ + jobId: 7010n, + waitingOn: head, + waitReason: "waiting_behind_pending", + }), + }, +}; + +export const SequenceUpNext: Story = { + args: { + job: pendingInSequence, + sequence: sequenceStatus({ waitReason: "up_next" }), + }, +}; + +const haltingJobStatus = sequenceStatus({ + halting: true, + jobId: 7001n, + jobState: JobState.Cancelled, + sequence: sequenceInspection({ + blockers: [halting], + haltedBy: halting, + head, + pendingCount: { count: 2341, more: false }, + }), +}); + +export const SequenceHaltingJob: Story = { + args: { + job: inSequence(jobFactory.cancelled().build({ id: 7001n })), + sequence: haltingJobStatus, + }, +}; + +// Deleting a halting job warns that the jobs behind it run without it, and +// offers Retry instead. +export const SequenceHaltingJobDelete: Story = { + args: SequenceHaltingJob.args, + play: async ({ canvas, userEvent }) => { + await userEvent.click(canvas.getByRole("button", { name: "Delete" })); + }, +}; + +export const SequenceNextPending: Story = { + args: { + job: inSequence(jobFactory.completed().build({ id: 6996n })), + sequence: sequenceStatus({ + jobId: 6996n, + jobState: JobState.Completed, + sequence: sequenceInspection({ head }), + }), + }, +}; + +export const SequenceNoPendingJobs: Story = { + args: { + job: inSequence(jobFactory.completed().build({ id: 7022n })), + sequence: sequenceStatus({ + jobId: 7022n, + jobState: JobState.Completed, + }), + }, +}; + +export const SequenceLoading: Story = { + args: { + job: pendingInSequence, + sequence: { error: false, key: sequenceKeyFixture }, + }, +}; + +export const SequenceUnavailable: Story = { + args: { + job: pendingInSequence, + sequence: { error: true, key: sequenceKeyFixture }, + }, +}; diff --git a/src/components/JobDetail.tsx b/src/components/JobDetail.tsx index ce26a29e..de64f51c 100644 --- a/src/components/JobDetail.tsx +++ b/src/components/JobDetail.tsx @@ -1,8 +1,9 @@ import { Badge } from "@components/Badge"; import ButtonForGroup from "@components/ButtonForGroup"; -import ConfirmationDialog from "@components/ConfirmationDialog"; import CopyableHeadingText from "@components/CopyableHeadingText"; +import DeleteJobConfirmation from "@components/DeleteJobConfirmation"; import JobAttempts from "@components/JobAttempts"; +import { SequenceStatusField } from "@components/JobSequenceStatus"; import JobTimeline from "@components/JobTimeline"; import JSONTextView from "@components/JSONTextView"; import JSONView from "@components/JSONView"; @@ -15,6 +16,10 @@ import { } from "@heroicons/react/24/outline"; import useFeature from "@hooks/use-feature"; import { Job, JobWithKnownMetadata } from "@services/jobs"; +import { + type JobSequenceStatus, + sequenceWaitSummary, +} from "@services/sequences"; import { JobState } from "@services/types"; import { Link } from "@tanstack/react-router"; import { capitalize } from "@utils/string"; @@ -25,6 +30,8 @@ type JobDetailProps = { deleteFn: () => void; job: Job; retry: () => void; + // Set for jobs in a sequence when the server supports sequence queries. + sequence?: JobSequenceStatus; }; type JobDetailViewProps = { @@ -32,6 +39,9 @@ type JobDetailViewProps = { job: Job; // Shown under the heading, for context that applies to the whole job. notice?: ReactNode; + // Set for jobs in a sequence. Adds a Sequence details field and a reason on + // a pending job's wait step. + sequence?: JobSequenceStatus; title: string; }; @@ -40,6 +50,7 @@ export default function JobDetail({ deleteFn, job, retry, + sequence, }: JobDetailProps) { return ( } job={job} + sequence={sequence} title="Job Details" /> ); @@ -63,6 +76,7 @@ export function JobDetailView({ actions, job, notice, + sequence, title, }: JobDetailViewProps) { const featureEnabledWorkflows = useFeature("ENABLE_WORKFLOWS", true); @@ -160,6 +174,13 @@ export function JobDetailView({ + {sequence && ( + + )} {featureEnabledWorkflows && jobWithMetadata && jobWithMetadata.metadata.workflow_id && ( @@ -187,7 +208,7 @@ export function JobDetailView({ - +
@@ -217,7 +238,13 @@ export function JobDetailView({ ); } -function ActionButtons({ cancel, deleteFn, job, retry }: JobDetailProps) { +function ActionButtons({ + cancel, + deleteFn, + job, + retry, + sequence, +}: JobDetailProps) { const [deleteConfirmationOpen, setDeleteConfirmationOpen] = useState(false); // Can only delete jobs that aren't running: @@ -276,19 +303,16 @@ function ActionButtons({ cancel, deleteFn, job, retry }: JobDetailProps) { text="Delete" /> - - This permanently deletes job{" "} - {job.id.toString()}. This action - cannot be undone. - - } + setDeleteConfirmationOpen(false)} - onConfirm={confirmDelete} + onDelete={confirmDelete} + onRetry={() => { + setDeleteConfirmationOpen(false); + retry(); + }} open={deleteConfirmationOpen} - title="Delete job?" + sequence={sequence} /> ); diff --git a/src/components/JobSequenceStatus.test.tsx b/src/components/JobSequenceStatus.test.tsx new file mode 100644 index 00000000..b961dbdd --- /dev/null +++ b/src/components/JobSequenceStatus.test.tsx @@ -0,0 +1,198 @@ +import type { JobSequence, JobSequenceStatus } from "@services/sequences"; + +import { JobState } from "@services/types"; +import { + jobSequence, + sequenceInspection, + sequenceJob, + sequenceKeyFixture, +} from "@test/factories/sequence"; +import { renderWithRouter } from "@test/utils/router"; +import { visibleText } from "@test/utils/text"; +import { screen } from "@testing-library/react"; +import { describe, expect, test } from "vitest"; + +import { SequenceStatusField } from "./JobSequenceStatus"; + +const halting = sequenceJob(7001n, JobState.Cancelled); +const running = sequenceJob(7002n, JobState.Running); +const head = sequenceJob(7003n, JobState.Pending); + +const status = (overrides: Partial = {}): JobSequenceStatus => ({ + error: false, + jobSequence: jobSequence(overrides), + key: sequenceKeyFixture, +}); + +const renderField = ( + job: { id: bigint; state: JobState }, + sequence: JobSequenceStatus, +) => + renderWithRouter(() => ( +
+ +
+ )); + +const pendingJob = { id: 7010n, state: JobState.Pending }; + +const fieldText = () => visibleText(screen.getByRole("definition")); + +describe("SequenceStatusField", () => { + test("links to the sequence page with the job selected", async () => { + await renderField(pendingJob, status({ waitReason: "up_next" })); + const link = screen.getByRole("link", { name: "View sequence" }); + expect(link).toHaveAttribute( + "href", + `/sequences/${encodeURIComponent(sequenceKeyFixture)}?job=7010`, + ); + }); + + test("names the job halting a pending job's sequence", async () => { + await renderField( + pendingJob, + status({ + sequence: sequenceInspection({ haltedBy: halting, head }), + waitingOn: halting, + waitReason: "halted", + }), + ); + expect(screen.getByText("Halted")).toBeInTheDocument(); + expect(fieldText()).toContain("by #7001 · Cancelled"); + expect(screen.getByText("Retry or delete it to resume.")).toBeVisible(); + expect(screen.getByRole("link", { name: "#7001" })).toHaveAttribute( + "href", + "/jobs/7001", + ); + // "by #7001" stays together when a narrow field wraps. + expect( + screen.getByRole("link", { name: "#7001" }).parentElement, + ).toHaveClass("whitespace-nowrap"); + }); + + test("names the active job a pending job waits on", async () => { + await renderField( + pendingJob, + status({ waitingOn: running, waitReason: "waiting_on_active" }), + ); + expect(screen.getByText("Waiting")).toBeInTheDocument(); + expect(fieldText()).toContain("on #7002 · Running"); + }); + + test("says when a retryable blocker runs again", async () => { + await renderField( + pendingJob, + status({ + waitingOn: sequenceJob(7002n, JobState.Retryable, { + scheduledAt: new Date(Date.now() + 4 * 60_000 + 5_000), + }), + waitReason: "waiting_on_active", + }), + ); + expect(fieldText()).toContain("Retrying in 4 minutes"); + }); + + test("says when a retry is overdue or a blocker is scheduled", async () => { + const { unmount } = await renderField( + pendingJob, + status({ + waitingOn: sequenceJob(7002n, JobState.Retryable, { + scheduledAt: new Date(Date.now() - 2 * 60_000 - 5_000), + }), + waitReason: "waiting_on_active", + }), + ); + expect(fieldText()).toContain("Retry due 2 minutes ago"); + unmount(); + + await renderField( + pendingJob, + status({ + waitingOn: sequenceJob(7002n, JobState.Scheduled, { + scheduledAt: new Date(Date.now() + 2 * 60 * 60_000 + 5_000), + }), + waitReason: "waiting_on_active", + }), + ); + expect(fieldText()).toContain("on #7002 · Runs in about 2 hours"); + }); + + test("names the earlier pending job a job waits behind", async () => { + await renderField( + pendingJob, + status({ waitingOn: head, waitReason: "waiting_behind_pending" }), + ); + expect(fieldText()).toContain("behind #7003 · Pending"); + }); + + test("says when nothing blocks a pending job", async () => { + await renderField(pendingJob, status({ waitReason: "up_next" })); + expect(screen.getByText("Up next")).toBeInTheDocument(); + expect(fieldText()).toContain("nothing blocking"); + }); + + test("says a halting job holds back the jobs after it", async () => { + await renderField( + { id: 7001n, state: JobState.Cancelled }, + status({ + halting: true, + jobState: JobState.Cancelled, + sequence: sequenceInspection({ haltedBy: halting, head }), + }), + ); + expect(screen.getByText("Halted")).toBeInTheDocument(); + expect(fieldText()).toContain("#7003 and later jobs wait"); + expect( + screen.getByText("Retry or delete this job to resume."), + ).toBeInTheDocument(); + }); + + test("points out an earlier job that also halts the sequence", async () => { + const earlier = sequenceJob(6998n, JobState.Discarded); + await renderField( + { id: 7001n, state: JobState.Cancelled }, + status({ + halting: true, + jobState: JobState.Cancelled, + sequence: sequenceInspection({ haltedBy: earlier, head }), + }), + ); + expect(fieldText()).toContain("Earlier job #6998 also halts it."); + }); + + test("links other jobs to the next pending job", async () => { + await renderField( + { id: 6996n, state: JobState.Completed }, + status({ + jobState: JobState.Completed, + sequence: sequenceInspection({ head }), + }), + ); + expect(fieldText()).toBe("Next pending: #7003"); + }); + + test("says when nothing is pending", async () => { + await renderField( + { id: 6996n, state: JobState.Completed }, + status({ jobState: JobState.Completed }), + ); + expect(fieldText()).toBe("No pending jobs"); + }); + + test("shows loading and failed states", async () => { + const { unmount } = await renderField(pendingJob, { + error: false, + key: sequenceKeyFixture, + }); + expect(fieldText()).toBe("Loading…"); + unmount(); + + await renderField(pendingJob, { + error: true, + jobSequence: jobSequence({ waitReason: "up_next" }), + key: sequenceKeyFixture, + }); + // A failed refresh doesn't present stale data. + expect(fieldText()).toBe("Status unavailable"); + }); +}); diff --git a/src/components/JobSequenceStatus.tsx b/src/components/JobSequenceStatus.tsx new file mode 100644 index 00000000..22c09d1c --- /dev/null +++ b/src/components/JobSequenceStatus.tsx @@ -0,0 +1,274 @@ +import { Badge, type BadgeColor } from "@components/Badge"; +import RelativeTimeFormatter from "@components/RelativeTimeFormatter"; +import { ArrowRightIcon } from "@heroicons/react/16/solid"; +import { + type JobSequence, + type JobSequenceStatus, + type SequenceJob, +} from "@services/sequences"; +import { JobState } from "@services/types"; +import { Link } from "@tanstack/react-router"; +import { capitalize } from "@utils/string"; +import clsx from "clsx"; +import { type ReactNode } from "react"; +import { useTime } from "react-time-sync"; + +type JobRef = { id: bigint; state: JobState }; + +type JobTiming = { scheduledAt: Date; state: JobState }; + +// SequenceJobLink links to a job in a sequence by ID, with its kind, queue, +// and state on hover. +export function SequenceJobLink({ + job, +}: { + job: { id: bigint } & Partial>; +}) { + return ( + + #{job.id.toString()} + + ); +} + +// SequenceJobStatePhrase says what happened to a job holding its sequence +// back, with when it runs next if that's known: "Cancelled", "Retrying in 4 +// minutes". A retry or schedule that's already due says so. +export function SequenceJobStatePhrase({ job }: { job: JobTiming }) { + const nowSec = useTime(); + const due = job.scheduledAt.getTime() <= nowSec * 1000; + const time = ; + switch (job.state) { + case JobState.Retryable: + return due ? <>Retry due {time} : <>Retrying {time}; + case JobState.Scheduled: + return due ? <>Due {time} : <>Runs {time}; + default: + return <>{capitalize(job.state)}; + } +} + +// Text colors match the state icons used across the UI. A retrying job is +// amber because it errored, and finalized failures are red. +const stateTextColors: Record = { + [JobState.Available]: "text-blue-700 dark:text-blue-400", + [JobState.Cancelled]: "text-red-700 dark:text-red-400", + [JobState.Completed]: "text-green-700 dark:text-green-400", + [JobState.Discarded]: "text-red-700 dark:text-red-400", + [JobState.Pending]: "text-slate-600 dark:text-slate-400", + [JobState.Retryable]: "text-amber-700 dark:text-amber-400", + [JobState.Running]: "text-blue-700 dark:text-blue-400", + [JobState.Scheduled]: "text-slate-600 dark:text-slate-400", +}; + +// SequenceJobRefState links to a job and follows it with its state as plain +// text, like "by #7001 · Cancelled", so the status badge and the link stay +// the line's only accents. The dot stays with the link, so a narrow field +// breaks before the state rather than before the dot. +export function SequenceJobRefState({ + job, + prefix, +}: { + job: SequenceJob; + prefix?: string; +}) { + return ( + <> + + {prefix && `${prefix} `} + + · + {" "} + + + + + ); +} + +// SequenceJobState is a job's state as a phrase in the state's color, for +// where the phrase stands on its own, like a sequence page row whose state +// icon is already beside it. +export function SequenceJobState({ job }: { job: JobTiming }) { + return ( + + + + ); +} + +const muted = "text-slate-500 dark:text-slate-400"; + +type StatusSummary = { + color: BadgeColor; + detail: ReactNode; + // A short follow-up line, like what to do about a halted sequence. + hint?: ReactNode; + label: string; +}; + +// SequenceStatusField is a details grid cell carrying a job's place in its +// sequence at a glance, with a link to the sequence's own page. +export function SequenceStatusField({ + className, + job, + status, +}: { + className?: string; + job: JobRef; + status: JobSequenceStatus; +}) { + const { jobSequence } = status; + const summary = + !status.error && jobSequence + ? sequenceStatusSummary(job, jobSequence) + : null; + const head = jobSequence?.sequence.head; + + let content: ReactNode; + if (status.error) { + content = Status unavailable; + } else if (!jobSequence) { + content = Loading…; + } else if (summary) { + content = ( + <> + + {summary.label} + {summary.detail} + + {summary.hint && ( + + {summary.hint} + + )} + + ); + } else if (head && head.id !== job.id) { + content = ( + <> + Next pending: + + ); + } else { + content = No pending jobs; + } + + return ( +
+
+ Sequence + + View +
+
+ {content} +
+
+ ); +} + +// Phrase keeps a short run of words, like "by #48213171", on one line, so a +// narrow field wraps between phrases rather than inside one. +function Phrase({ + children, + className, +}: { + children: ReactNode; + className?: string; +}) { + return ( + {children} + ); +} + +// sequenceStatusSummary is the badge and explanation for a job in a sequence. +// Pending jobs say why they're waiting, and a job that halts its sequence says +// so. Other jobs get no summary and show the sequence's next pending job +// instead. +function sequenceStatusSummary( + job: JobRef, + jobSequence: JobSequence, +): null | StatusSummary { + const { sequence, waitingOn } = jobSequence; + + if (job.state === JobState.Pending) { + switch (jobSequence.waitReason) { + case "halted": + if (!waitingOn) break; + return { + color: "red", + detail: , + hint: "Retry or delete it to resume.", + label: "Halted", + }; + case "none": + break; + case "up_next": + return { + color: "zinc", + detail: nothing blocking, + label: "Up next", + }; + case "waiting_behind_pending": + if (!waitingOn) break; + return { + color: "amber", + detail: , + label: "Waiting", + }; + case "waiting_on_active": + if (!waitingOn) break; + return { + color: "amber", + detail: , + label: "Waiting", + }; + } + return null; + } + + if (jobSequence.halting && sequence.head) { + const earlierHalt = + sequence.haltedBy && sequence.haltedBy.id !== job.id + ? sequence.haltedBy + : null; + return { + color: "red", + detail: ( + <> + {" "} + and later jobs wait + + ), + hint: earlierHalt ? ( + <> + Earlier job also halts it. + + ) : ( + "Retry or delete this job to resume." + ), + label: "Halted", + }; + } + + return null; +} diff --git a/src/components/JobTimeline.test.tsx b/src/components/JobTimeline.test.tsx index d1563057..058a2749 100644 --- a/src/components/JobTimeline.test.tsx +++ b/src/components/JobTimeline.test.tsx @@ -44,6 +44,35 @@ const getStepNames = (): string[] => { }; describe("JobTimeline", () => { + it("names and tints a pending job's wait reason", () => { + const job = jobFactory.pending().build(); + const { rerender } = render(); + expect( + within(getStepItem("Wait")).getByText("Pending"), + ).toBeInTheDocument(); + expect(getStepIcon("Wait")).toHaveClass("bg-slate-100"); + + rerender( + , + ); + expect(screen.getByText("Sequence halted")).toHaveClass("text-red-700"); + expect(getStepIcon("Wait")).toHaveClass("bg-red-200"); + + rerender( + , + ); + expect(screen.getByText("Waiting on sequence")).toHaveClass( + "text-amber-700", + ); + expect(getStepIcon("Wait")).toHaveClass("bg-amber-200"); + }); + it("renders snoozed jobs with a dedicated snoozed step after running", () => { // This fixture models a job that already ran once, then was snoozed back // into `scheduled`. In that state, the next retry time is known, but the diff --git a/src/components/JobTimeline.tsx b/src/components/JobTimeline.tsx index 4c6a9ff8..9ec0d7db 100644 --- a/src/components/JobTimeline.tsx +++ b/src/components/JobTimeline.tsx @@ -11,6 +11,7 @@ import { XCircleIcon, } from "@heroicons/react/24/solid"; import { AttemptError, Job } from "@services/jobs"; +import { type SequenceWaitTone } from "@services/sequences"; import { Heroicon, JobState } from "@services/types"; import clsx from "clsx"; import { formatDistanceStrict } from "date-fns"; @@ -188,7 +189,23 @@ const ScheduledStep = ({ job }: { job: Job }) => { ); }; -const WaitStep = ({ job }: { job: Job }) => { +// A pending job's wait step names what it's waiting on when that's known, +// like its sequence, tinted by whether it needs attention. +export type WaitSummary = { label: string; tone: SequenceWaitTone }; + +const waitToneStatus: Record = { + halted: "failed", + neutral: "pending", + waiting: "waiting", +}; + +const waitToneTextClasses: Record = { + halted: "font-medium text-red-700 dark:text-red-400", + neutral: "", + waiting: "font-medium text-amber-700 dark:text-amber-400", +}; + +const WaitStep = ({ job, wait }: { job: Job; wait?: WaitSummary }) => { const nowSec = useTime(); const now = useMemo(() => new Date(nowSec * 1000), [nowSec]); const scheduledAtInFuture = useMemo(() => job.scheduledAt >= now, [job, now]); @@ -220,11 +237,16 @@ const WaitStep = ({ job }: { job: Job }) => { if (job.state === JobState.Pending) { return ( + status={wait ? waitToneStatus[wait.tone] : "pending"} + > + {wait ? ( + {wait.label} + ) : ( + "Pending" + )} + ); } @@ -435,9 +457,12 @@ const FinalizedStep = ({ job }: { job: Job }) => { type JobTimelineProps = { job: Job; + // Replaces a pending job's "Pending" wait step with a more specific reason, + // like a job held back by its sequence. + wait?: WaitSummary; }; -export default function JobTimeline({ job }: JobTimelineProps) { +export default function JobTimeline({ job, wait }: JobTimelineProps) { return (
    - + diff --git a/src/components/SequenceDetail.stories.tsx b/src/components/SequenceDetail.stories.tsx new file mode 100644 index 00000000..09a9bc0d --- /dev/null +++ b/src/components/SequenceDetail.stories.tsx @@ -0,0 +1,112 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; + +import { JobState } from "@services/types"; +import { + haltedSequenceFixture, + sequenceInspection, + sequenceJob, + sequenceKeyFixture, + sequenceListItem, +} from "@test/factories/sequence"; +import { fn } from "storybook/test"; + +import SequenceDetail from "./SequenceDetail"; + +const meta: Meta = { + args: { + hasEarlier: false, + hasLater: false, + onJumpToJob: fn(), + onRemoveHalt: fn(), + onShowEarlier: fn(), + onShowLater: fn(), + sequenceKey: sequenceKeyFixture, + }, + component: SequenceDetail, + title: "Pages/SequenceDetail", +}; + +export default meta; + +type Story = StoryObj; + +// A cancelled job halts the sequence. The page opens at the halting job, with +// earlier completed jobs behind "Show earlier", and outlines the job the +// operator came from. +export const Halted: Story = { + args: { + hasEarlier: true, + hasLater: true, + inspection: haltedSequenceFixture.inspection, + jobs: haltedSequenceFixture.jobs, + selectedJobId: 7003n, + }, +}; + +// Deleting the halting job from the sequence page shows the same warning as +// the job page. +export const HaltedDelete: Story = { + args: Halted.args, + play: async ({ canvas, userEvent }) => { + await userEvent.click(canvas.getByRole("button", { name: "Delete" })); + }, +}; + +const running = sequenceJob(7002n, JobState.Running); + +export const WaitingOnRunningJob: Story = { + args: { + hasEarlier: true, + inspection: sequenceInspection({ + blockers: [running], + head: sequenceJob(7003n, JobState.Pending), + pendingCount: { count: 2, more: false }, + }), + jobs: [ + sequenceListItem(7002n, JobState.Running, 2, "packed"), + sequenceListItem(7003n, JobState.Pending, 300, "shipped"), + sequenceListItem(7010n, JobState.Pending, 120, "delivered"), + ], + selectedJobId: 7010n, + }, +}; + +export const UpNext: Story = { + args: { + hasEarlier: true, + inspection: sequenceInspection({ + head: sequenceJob(7003n, JobState.Pending), + pendingCount: { count: 1, more: false }, + }), + jobs: [sequenceListItem(7003n, JobState.Pending, 1, "shipped")], + }, +}; + +// Nothing is pending or running, so the page shows the newest jobs. +export const Idle: Story = { + args: { + hasEarlier: true, + inspection: sequenceInspection(), + jobs: [ + sequenceListItem(6990n, JobState.Completed, 1440, "placed"), + sequenceListItem(6996n, JobState.Completed, 900, "paid"), + sequenceListItem(7001n, JobState.Completed, 420, "shipped"), + ], + selectedJobId: 6996n, + }, +}; + +// The job the operator came from is outside the loaded range. +export const SelectedJobNotLoaded: Story = { + args: { + ...Halted.args, + selectedJobId: 6912n, + }, +}; + +export const Loading: Story = { + args: { + jobs: [], + jobsLoading: true, + }, +}; diff --git a/src/components/SequenceDetail.test.tsx b/src/components/SequenceDetail.test.tsx new file mode 100644 index 00000000..12e05af6 --- /dev/null +++ b/src/components/SequenceDetail.test.tsx @@ -0,0 +1,205 @@ +import { JobState } from "@services/types"; +import { + haltedSequenceFixture, + sequenceInspection, + sequenceJob, + sequenceKeyFixture, + sequenceListItem, +} from "@test/factories/sequence"; +import { renderWithRouter } from "@test/utils/router"; +import { visibleText } from "@test/utils/text"; +import { fireEvent, screen, within } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +import SequenceDetail, { type SequenceDetailProps } from "./SequenceDetail"; + +const renderPage = async (props: Partial = {}) => { + const handlers = { + onJumpToJob: vi.fn(), + onRemoveHalt: vi.fn(), + onShowEarlier: vi.fn(), + onShowLater: vi.fn(), + }; + const result = await renderWithRouter(() => ( + + )); + return { ...handlers, result }; +}; + +const row = (id: bigint) => { + const item = document.getElementById(`job-${id.toString()}`); + if (!item) throw new Error(`Expected a row for job ${id.toString()}`); + return item; +}; + +// fact reads a details grid field by its term. +const fact = (term: string) => + visibleText(screen.getByText(term, { selector: "dt" }).nextElementSibling); + +describe("SequenceDetail", () => { + test("titles the page by its jobs' kind, with a status and the key", async () => { + await renderPage(); + const header = screen.getByRole("banner"); + expect(within(header).getByRole("heading", { level: 1 })).toHaveTextContent( + "update_order_status", + ); + expect(within(header).getByText("Halted")).toBeInTheDocument(); + // The key is shortened on screen but kept whole for hover and copy. + expect(within(header).getByTitle(sequenceKeyFixture)).toHaveTextContent( + sequenceKeyFixture, + ); + expect( + within(header).getByRole("button", { name: "Copy sequence key" }), + ).toBeInTheDocument(); + }); + + test("falls back to a generic title for mixed kinds", async () => { + await renderPage({ + inspection: sequenceInspection(), + jobs: [ + sequenceListItem(6990n, JobState.Completed, 90, "placed"), + sequenceListItem(6996n, JobState.Completed, 60, "paid"), + ], + }); + expect( + within(screen.getByRole("banner")).getByRole("heading", { level: 1 }), + ).toHaveTextContent("Sequence"); + }); + + test("details why the sequence is halted", async () => { + await renderPage(); + expect(fact("Halted by")).toBe( + "#7001 · CancelledRetry or delete it to resume.", + ); + expect(fact("Next pending")).toBe("#7003"); + expect(fact("Pending")).toBe("2,341 jobs"); + }); + + test("shows each job's state as a titled icon and what a held-back job did", async () => { + await renderPage(); + expect(within(row(7001n)).getByTitle("Cancelled")).toBeInTheDocument(); + // The job holding the sequence back says what happened to it. + expect(visibleText(row(7001n))).toContain( + "Cancelled · halted the sequence", + ); + expect(within(row(7003n)).getByTitle("Pending")).toBeInTheDocument(); + // Pending jobs get no third line: the details say which runs next and + // what they wait on. + expect(visibleText(row(7003n))).not.toContain("·"); + expect( + within(row(7001n)).getByRole("link", { name: "#7001" }), + ).toHaveAttribute("href", "/jobs/7001"); + }); + + test("outlines the job the operator came from", async () => { + await renderPage({ selectedJobId: 7010n }); + expect(row(7010n)).toHaveAttribute("aria-current", "true"); + // The outline is the only marker; the page has no "this" to name. + expect(row(7010n)).toHaveClass("ring-2"); + expect(row(7003n)).not.toHaveAttribute("aria-current"); + }); + + test("offers to show a selected job outside the loaded range", async () => { + const { onJumpToJob } = await renderPage({ selectedJobId: 6912n }); + fireEvent.click(screen.getByRole("button", { name: "Show it" })); + expect(onJumpToJob).toHaveBeenCalledWith(6912n); + }); + + test("retries the halting job in place", async () => { + const { onRemoveHalt } = await renderPage(); + fireEvent.click(within(row(7001n)).getByRole("button", { name: "Retry" })); + expect(onRemoveHalt).toHaveBeenCalledWith(7001n, "retry"); + }); + + test("warns before deleting the halting job, and can retry instead", async () => { + const { onRemoveHalt } = await renderPage(); + fireEvent.click(within(row(7001n)).getByRole("button", { name: "Delete" })); + expect(screen.getByRole("dialog")).toHaveTextContent( + "2,341 pending jobs are waiting behind it", + ); + fireEvent.click(screen.getByRole("button", { name: "Retry instead" })); + expect(onRemoveHalt).toHaveBeenCalledWith(7001n, "retry"); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + + fireEvent.click(within(row(7001n)).getByRole("button", { name: "Delete" })); + fireEvent.click(screen.getByRole("button", { name: "Delete anyway" })); + expect(onRemoveHalt).toHaveBeenCalledWith(7001n, "delete"); + }); + + test("sizes ID links to the longest ID so kinds line up", async () => { + await renderPage({ + jobs: [ + sequenceListItem(99n, JobState.Completed, 60, "done"), + sequenceListItem(100n, JobState.Pending, 30, "next"), + ], + }); + for (const id of [99n, 100n]) { + expect( + within(row(id)).getByRole("link", { name: `#${id.toString()}` }), + ).toHaveStyle({ minWidth: "4ch" }); + } + }); + + test("pages toward earlier and later jobs", async () => { + const { onShowEarlier, onShowLater } = await renderPage({ + hasEarlier: true, + hasLater: true, + }); + fireEvent.click(screen.getByRole("button", { name: "Show earlier jobs" })); + expect(onShowEarlier).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByRole("button", { name: "Show later jobs" })); + expect(onShowLater).toHaveBeenCalledTimes(1); + }); + + test("hides paging past either end", async () => { + await renderPage(); + expect( + screen.queryByRole("button", { name: "Show earlier jobs" }), + ).not.toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: "Show later jobs" }), + ).not.toBeInTheDocument(); + }); + + test("details a sequence waiting on a running job", async () => { + await renderPage({ + inspection: sequenceInspection({ + blockers: [sequenceJob(7002n, JobState.Running)], + head: sequenceJob(7003n, JobState.Pending), + pendingCount: { count: 1, more: false }, + }), + jobs: [ + sequenceListItem(7002n, JobState.Running, 2, "sent"), + sequenceListItem(7003n, JobState.Pending, 30, "shipped"), + ], + }); + expect(screen.getByRole("banner")).toHaveTextContent("Waiting"); + expect(fact("Waiting on")).toBe("#7002 · Running"); + expect(fact("Pending")).toBe("1 job"); + expect(visibleText(row(7002n))).toContain( + "Running · blocking the sequence", + ); + expect( + screen.queryByRole("button", { name: "Retry" }), + ).not.toBeInTheDocument(); + }); + + test("details an idle sequence", async () => { + await renderPage({ + inspection: sequenceInspection(), + jobs: [sequenceListItem(7001n, JobState.Completed, 30, "done")], + }); + expect(screen.getByRole("banner")).toHaveTextContent("Idle"); + expect(fact("Next pending")).toBe("–"); + expect(fact("Pending")).toBe("–"); + expect(screen.queryByText("Halted by")).not.toBeInTheDocument(); + }); +}); diff --git a/src/components/SequenceDetail.tsx b/src/components/SequenceDetail.tsx new file mode 100644 index 00000000..eeabd2c3 --- /dev/null +++ b/src/components/SequenceDetail.tsx @@ -0,0 +1,483 @@ +import { Badge, type BadgeColor } from "@components/Badge"; +import CopyableHeadingText from "@components/CopyableHeadingText"; +import HaltingJobDeleteDialog from "@components/HaltingJobDeleteDialog"; +import { + SequenceJobLink, + SequenceJobRefState, + SequenceJobState, +} from "@components/JobSequenceStatus"; +import RelativeTimeFormatter from "@components/RelativeTimeFormatter"; +import { TaskStateIcon } from "@components/TaskStateIcon"; +import TopNavTitleOnly from "@components/TopNavTitleOnly"; +import { + ArrowDownIcon, + ArrowUpIcon, + ArrowUturnLeftIcon, + TrashIcon, +} from "@heroicons/react/24/outline"; +import { + formatPendingCount, + isActiveState, + type SequenceInspection, + type SequenceJob, + type SequenceJobListItem, +} from "@services/sequences"; +import { JobState } from "@services/types"; +import { Link } from "@tanstack/react-router"; +import { compactJSONText } from "@utils/jsonText"; +import clsx from "clsx"; +import { type ReactNode, useMemo, useState } from "react"; + +export type SequenceDetailProps = { + hasEarlier: boolean; + hasLater: boolean; + inspection?: SequenceInspection; + inspectionError?: boolean; + jobs: SequenceJobListItem[]; + jobsError?: boolean; + jobsLoading?: boolean; + // Loads the job's surroundings when it isn't in the loaded range. + onJumpToJob?: (jobId: bigint) => void; + onRemoveHalt: (jobId: bigint, action: "delete" | "retry") => void; + onShowEarlier: () => void; + onShowLater: () => void; + // The job the operator came from, outlined in the list. + selectedJobId?: bigint; + sequenceKey: string; + showingEarlier?: boolean; + showingLater?: boolean; +}; + +const muted = "text-slate-500 dark:text-slate-400"; + +export default function SequenceDetail({ + hasEarlier, + hasLater, + inspection, + inspectionError, + jobs, + jobsError, + jobsLoading, + onJumpToJob, + onRemoveHalt, + onShowEarlier, + onShowLater, + selectedJobId, + sequenceKey, + showingEarlier, + showingLater, +}: SequenceDetailProps) { + const [deleteOpen, setDeleteOpen] = useState(false); + const haltedBy = inspection?.haltedBy; + const head = inspection?.head; + // IDs are monospace, so `ch` fits the longest one plus its "#". + const idWidth = `${Math.max(0, ...jobs.map((job) => job.id.toString().length)) + 1}ch`; + const selectedLoaded = + selectedJobId === undefined || jobs.some((job) => job.id === selectedJobId); + + return ( + <> + +
    +
    +
    +

    + {sequenceTitle(jobs, inspection)} +

    + + {inspectionError ? ( + Status unavailable + ) : inspection ? ( + + ) : null} + +
    + {/* The key is a hash, useful only to copy, so it's shortened like + a commit SHA. */} +

    + Key: + + + +

    +
    + +
    + {inspection && !inspectionError && ( + + )} + +
    +

    + Jobs +

    +

    + Jobs in a sequence run one at a time, in ID order. +

    +
    + + {!selectedLoaded && selectedJobId !== undefined && !jobsLoading && ( +

    + Job #{selectedJobId.toString()}{" "} + isn't in the jobs shown below.{" "} + {onJumpToJob && ( + + )} +

    + )} + +
      + {hasEarlier && ( + + )} + {jobs.map((job) => ( + setDeleteOpen(true)} + onRetry={() => onRemoveHalt(job.id, "retry")} + selected={job.id === selectedJobId} + /> + ))} + {jobs.length === 0 && ( +
    1. + {jobsError + ? "Couldn't load this sequence's jobs." + : jobsLoading + ? "Loading jobs…" + : "No jobs in this sequence."} +
    2. + )} + {hasLater && ( + + )} +
    +
    +
    + + {haltedBy && head && inspection?.pendingCount && ( + setDeleteOpen(false)} + onDelete={() => { + setDeleteOpen(false); + onRemoveHalt(haltedBy.id, "delete"); + }} + onRetry={() => { + setDeleteOpen(false); + onRemoveHalt(haltedBy.id, "retry"); + }} + open={deleteOpen} + pendingCount={inspection.pendingCount} + /> + )} + + ); +} + +// sequenceTitle names a sequence by the kind of its jobs, like the job page. +// The key is a hash of options that aren't stored, so the kinds seen in the +// loaded jobs are all that can be shown reliably. Mixed kinds, or none loaded +// yet, fall back to a generic title. +const sequenceTitle = ( + jobs: { kind: string }[], + inspection?: SequenceInspection, +): string => { + const kinds = new Set(jobs.map((job) => job.kind)); + for (const job of [ + inspection?.haltedBy, + inspection?.head, + ...(inspection?.blockers ?? []), + ]) { + if (job) kinds.add(job.kind); + } + const [only] = kinds; + return kinds.size === 1 && only ? only : "Sequence"; +}; + +// sequenceStatus names the state of the whole sequence: halted by a job, +// waiting on an active job, about to run its next job, or idle. +const sequenceStatus = ( + inspection: SequenceInspection, +): { color: BadgeColor; label: string } => { + const active = inspection.blockers.some((job) => isActiveState(job.state)); + if (inspection.haltedBy) return { color: "red", label: "Halted" }; + if (inspection.head && active) return { color: "amber", label: "Waiting" }; + if (inspection.head) return { color: "zinc", label: "Up next" }; + if (active) return { color: "blue", label: "Active" }; + return { color: "zinc", label: "Idle" }; +}; + +function SequenceStatusBadge({ + inspection, +}: { + inspection: SequenceInspection; +}) { + const { color, label } = sequenceStatus(inspection); + return {label}; +} + +const factClassName = + "border-t border-slate-100 py-4 pr-4 sm:col-span-4 dark:border-slate-800"; + +// SequenceFacts is the sequence's details grid: the job holding it back, if +// any, its next pending job, and how many jobs are pending. +export function SequenceFacts({ + inspection, +}: { + inspection: SequenceInspection; +}) { + const { haltedBy, head, pendingCount } = inspection; + const active = inspection.blockers.find((job) => isActiveState(job.state)); + + let blocker: ReactNode = null; + if (haltedBy) { + blocker = ( + + + + ); + } else if (active) { + blocker = ( + + + + ); + } + + return ( +
    + {blocker} + + {head ? : "–"} + + + {pendingCount && pendingCount.count > 0 + ? `${formatPendingCount(pendingCount)} ${pendingCount.count === 1 && !pendingCount.more ? "job" : "jobs"}` + : "–"} + +
    + ); +} + +function Fact({ + children, + className = "col-span-6", + hint, + term, +}: { + children: ReactNode; + className?: string; + hint?: ReactNode; + term: string; +}) { + return ( +
    +
    + {term} +
    +
    + {children} + {hint && ( + + {hint} + + )} +
    +
    + ); +} + +function PagingRow({ + Icon, + loading, + onClick, + text, +}: { + Icon: typeof ArrowUpIcon; + loading?: boolean; + onClick: () => void; + text: string; +}) { + return ( +
  1. + +
  2. + ); +} + +// rowTime is the moment a row's time column describes, like the Jobs list: +// when a finished job finalized, when an active job runs next or started, and +// when a pending job was added. +const rowTime = (job: SequenceJobListItem): Date => { + switch (job.state) { + case JobState.Available: + case JobState.Retryable: + case JobState.Scheduled: + return job.scheduledAt; + case JobState.Cancelled: + case JobState.Completed: + case JobState.Discarded: + return job.finalizedAt ?? job.createdAt; + case JobState.Pending: + return job.createdAt; + case JobState.Running: + return job.attemptedAt ?? job.scheduledAt; + } +}; + +// RowActionButton is a compact button for actions inside a list row. Page-level +// buttons grow at mobile widths, which would crowd a row. +function RowActionButton({ + Icon, + onClick, + text, +}: { + Icon: typeof TrashIcon; + onClick: () => void; + text: string; +}) { + return ( + + ); +} + +function SequenceJobRow({ + halting, + head, + idWidth, + inspection, + job, + onDelete, + onRetry, + selected, +}: { + halting: boolean; + head?: null | SequenceJob; + // Fits the longest ID in the list, so kinds line up. + idWidth: string; + inspection?: SequenceInspection; + job: SequenceJobListItem; + onDelete: () => void; + onRetry: () => void; + selected: boolean; +}) { + const id = job.id.toString(); + const blocking = + isActiveState(job.state) && + !!head && + !!inspection?.blockers.some((blocker) => blocker.id === job.id); + // What a job holding the sequence back did to it. + const role = halting + ? "halted the sequence" + : blocking + ? "blocking the sequence" + : null; + const args = useMemo(() => compactJSONText(job.argsRaw), [job.argsRaw]); + + return ( +
  3. +
    + + + #{id} + + + {job.kind} + + + + +
    + {/* Args, like the Jobs list. Phones hide them. */} +

    + {args} +

    + {/* Only a job holding the sequence back gets a third line: what + happened to it, and any actions to clear it. The state's icon is + already at the start of the row. */} + {role && ( +
    + + + · {role} + + {halting && ( + + + + + )} +
    + )} +
  4. + ); +} diff --git a/src/components/WorkflowDetail.test.tsx b/src/components/WorkflowDetail.test.tsx index 06afe36e..cea8d198 100644 --- a/src/components/WorkflowDetail.test.tsx +++ b/src/components/WorkflowDetail.test.tsx @@ -47,6 +47,7 @@ const features: Features = { hasSequenceTable: false, jobListHideArgsByDefault: false, producerQueries: true, + sequenceQueries: false, workflowQueries: true, }; diff --git a/src/routeTree.gen.ts b/src/routeTree.gen.ts index bbfec102..6f350839 100644 --- a/src/routeTree.gen.ts +++ b/src/routeTree.gen.ts @@ -22,6 +22,7 @@ import { Route as JobsJobIdRouteImport } from "./routes/jobs/$jobId" import { Route as PeriodicJobsIndexRouteImport } from "./routes/periodic-jobs/index" import { Route as QueuesIndexRouteImport } from "./routes/queues/index" import { Route as QueuesNameRouteImport } from "./routes/queues/$name" +import { Route as SequencesSequenceKeyRouteImport } from "./routes/sequences/$sequenceKey" import { Route as WorkflowsIndexRouteImport } from "./routes/workflows/index" import { Route as WorkflowsWorkflowIdRouteImport } from "./routes/workflows/$workflowId" @@ -90,6 +91,11 @@ const QueuesNameRoute = QueuesNameRouteImport.update({ path: "/$name", getParentRoute: () => QueuesRoute, } as any) +const SequencesSequenceKeyRoute = SequencesSequenceKeyRouteImport.update({ + id: "/sequences/$sequenceKey", + path: "/sequences/$sequenceKey", + getParentRoute: () => rootRouteImport, +} as any) const WorkflowsIndexRoute = WorkflowsIndexRouteImport.update({ id: "/workflows/", path: "/workflows/", @@ -111,6 +117,7 @@ export interface FileRoutesByFullPath { "/dead-letter-jobs/$jobId": typeof DeadLetterJobsJobIdRoute "/jobs/$jobId": typeof JobsJobIdRoute "/queues/$name": typeof QueuesNameRoute + "/sequences/$sequenceKey": typeof SequencesSequenceKeyRoute "/workflows/$workflowId": typeof WorkflowsWorkflowIdRoute "/dead-letter-jobs/": typeof DeadLetterJobsIndexRoute "/jobs/": typeof JobsIndexRoute @@ -125,6 +132,7 @@ export interface FileRoutesByTo { "/dead-letter-jobs/$jobId": typeof DeadLetterJobsJobIdRoute "/jobs/$jobId": typeof JobsJobIdRoute "/queues/$name": typeof QueuesNameRoute + "/sequences/$sequenceKey": typeof SequencesSequenceKeyRoute "/workflows/$workflowId": typeof WorkflowsWorkflowIdRoute "/dead-letter-jobs": typeof DeadLetterJobsIndexRoute "/jobs": typeof JobsIndexRoute @@ -143,6 +151,7 @@ export interface FileRoutesById { "/dead-letter-jobs/$jobId": typeof DeadLetterJobsJobIdRoute "/jobs/$jobId": typeof JobsJobIdRoute "/queues/$name": typeof QueuesNameRoute + "/sequences/$sequenceKey": typeof SequencesSequenceKeyRoute "/workflows/$workflowId": typeof WorkflowsWorkflowIdRoute "/dead-letter-jobs/": typeof DeadLetterJobsIndexRoute "/jobs/": typeof JobsIndexRoute @@ -162,6 +171,7 @@ export interface FileRouteTypes { | "/dead-letter-jobs/$jobId" | "/jobs/$jobId" | "/queues/$name" + | "/sequences/$sequenceKey" | "/workflows/$workflowId" | "/dead-letter-jobs/" | "/jobs/" @@ -176,6 +186,7 @@ export interface FileRouteTypes { | "/dead-letter-jobs/$jobId" | "/jobs/$jobId" | "/queues/$name" + | "/sequences/$sequenceKey" | "/workflows/$workflowId" | "/dead-letter-jobs" | "/jobs" @@ -193,6 +204,7 @@ export interface FileRouteTypes { | "/dead-letter-jobs/$jobId" | "/jobs/$jobId" | "/queues/$name" + | "/sequences/$sequenceKey" | "/workflows/$workflowId" | "/dead-letter-jobs/" | "/jobs/" @@ -208,6 +220,7 @@ export interface RootRouteChildren { QueuesRoute: typeof QueuesRouteWithChildren SettingsRoute: typeof SettingsRoute AboutAboutRoute: typeof AboutAboutRoute + SequencesSequenceKeyRoute: typeof SequencesSequenceKeyRoute WorkflowsWorkflowIdRoute: typeof WorkflowsWorkflowIdRoute PeriodicJobsIndexRoute: typeof PeriodicJobsIndexRoute WorkflowsIndexRoute: typeof WorkflowsIndexRoute @@ -306,6 +319,13 @@ declare module "@tanstack/react-router" { preLoaderRoute: typeof QueuesNameRouteImport parentRoute: typeof QueuesRoute } + "/sequences/$sequenceKey": { + id: "/sequences/$sequenceKey" + path: "/sequences/$sequenceKey" + fullPath: "/sequences/$sequenceKey" + preLoaderRoute: typeof SequencesSequenceKeyRouteImport + parentRoute: typeof rootRouteImport + } "/workflows/": { id: "/workflows/" path: "/workflows" @@ -369,6 +389,7 @@ const rootRouteChildren: RootRouteChildren = { QueuesRoute: QueuesRouteWithChildren, SettingsRoute: SettingsRoute, AboutAboutRoute: AboutAboutRoute, + SequencesSequenceKeyRoute: SequencesSequenceKeyRoute, WorkflowsWorkflowIdRoute: WorkflowsWorkflowIdRoute, PeriodicJobsIndexRoute: PeriodicJobsIndexRoute, WorkflowsIndexRoute: WorkflowsIndexRoute, diff --git a/src/routes/jobs/$jobId.tsx b/src/routes/jobs/$jobId.tsx index f3a8415f..2f47d000 100644 --- a/src/routes/jobs/$jobId.tsx +++ b/src/routes/jobs/$jobId.tsx @@ -1,12 +1,24 @@ import JobDetail from "@components/JobDetail"; import JobNotFound from "@components/JobNotFound"; +import { useFeatures } from "@contexts/Features.hook"; import { useRefreshSetting } from "@contexts/RefreshSettings.hook"; import { refreshQueryOptions } from "@contexts/RefreshSettings.query"; import { useRetryJobs } from "@hooks/use-retry-jobs"; import { cancelJobs, deleteJobs, getJob, getJobKey } from "@services/jobs"; +import { + getJobSequence, + jobSequenceKey, + sequenceKeyFromMetadata, + sequenceKeyPrefix, +} from "@services/sequences"; import { toastError } from "@services/toast"; import { JobState } from "@services/types"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + keepPreviousData, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; import { createFileRoute, ErrorComponent, @@ -47,7 +59,7 @@ export const Route = createFileRoute("/jobs/$jobId")({ component: JobComponent, }); -function JobComponent() { +export function JobComponent() { const { jobId } = Route.useParams(); const navigate = Route.useNavigate(); const { queryOptions } = Route.useRouteContext(); @@ -62,6 +74,40 @@ function JobComponent() { const queryClient = useQueryClient(); const jobQuery = useQuery(queryOptionsWithRefresh); + const { features } = useFeatures(); + const sequenceKey = jobQuery.data + ? sequenceKeyFromMetadata(jobQuery.data.metadata) + : undefined; + const sequenceEnabled = features.sequenceQueries && !!sequenceKey; + const jobState = jobQuery.data?.state; + // Only a pending job's wait reason changes while the job itself doesn't, so + // other jobs load their sequence once per job state instead of polling it. + // A halting job's own state changes when it's retried, which reloads it. + const sequenceRefresh = refreshQueryOptions( + jobState === JobState.Pending ? refreshSettings.intervalMs : 0, + ); + const sequenceQuery = useQuery({ + enabled: sequenceEnabled, + queryFn: getJobSequence, + queryKey: jobSequenceKey(jobId, jobState), + ...sequenceRefresh, + // Avoid flashing the field empty while a new state's snapshot loads. + placeholderData: keepPreviousData, + staleTime: sequenceRefresh.refetchInterval === false ? Infinity : 0, + }); + + const invalidateJob = () => + Promise.all([ + queryClient.invalidateQueries({ + queryKey: queryOptionsWithRefresh.queryKey, + }), + queryClient.invalidateQueries({ queryKey: jobSequenceKey(jobId) }), + // A cached sequence page shows this job too. + sequenceKey && + queryClient.invalidateQueries({ + queryKey: sequenceKeyPrefix(sequenceKey), + }), + ]); const cancelMutation = useMutation({ mutationFn: async (_variables, context) => @@ -72,9 +118,7 @@ function JobComponent() { message: "Job cancelled", duration: 2000, }); - return queryClient.invalidateQueries({ - queryKey: queryOptionsWithRefresh.queryKey, - }); + return invalidateJob(); }, }); @@ -95,11 +139,7 @@ function JobComponent() { }); const retryMutation = useRetryJobs({ - onSuccess: () => { - return queryClient.invalidateQueries({ - queryKey: queryOptionsWithRefresh.queryKey, - }); - }, + onSuccess: invalidateJob, successMessage: "Job enqueued for retry", }); @@ -115,6 +155,15 @@ function JobComponent() { deleteFn={deleteMutation.mutate} job={job} retry={() => retryMutation.mutate([jobId])} + sequence={ + sequenceEnabled && sequenceKey + ? { + error: sequenceQuery.isError, + jobSequence: sequenceQuery.data, + key: sequenceKey, + } + : undefined + } /> ); } diff --git a/src/routes/jobs/jobSequence.test.tsx b/src/routes/jobs/jobSequence.test.tsx new file mode 100644 index 00000000..d73c034e --- /dev/null +++ b/src/routes/jobs/jobSequence.test.tsx @@ -0,0 +1,143 @@ +import type { JobSequenceStatus } from "@services/sequences"; + +import { JobComponent, Route } from "@routes/jobs/$jobId"; +import { getJobKey } from "@services/jobs"; +import { JobState } from "@services/types"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { jobFactory } from "@test/factories/job"; +import { jobSequence } from "@test/factories/sequence"; +import { createFeatures } from "@test/utils/features"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +const { featureState, getJob, getSequence, refreshState } = vi.hoisted(() => ({ + featureState: { enabled: true }, + getJob: vi.fn(), + getSequence: vi.fn(), + refreshState: { intervalMs: 0 }, +})); + +vi.mock("@contexts/Features.hook", () => ({ + useFeatures: () => ({ + features: createFeatures({ sequenceQueries: featureState.enabled }), + }), +})); +vi.mock("@contexts/RefreshSettings.hook", () => ({ + useRefreshSetting: () => ({ intervalMs: refreshState.intervalMs }), +})); +vi.mock("@services/sequences", async (original) => ({ + ...(await original()), + getJobSequence: getSequence, +})); +vi.mock("@components/JobDetail", () => ({ + default: ({ + retry, + sequence, + }: { + retry: () => void; + sequence?: JobSequenceStatus; + }) => ( +
    + Job details + {sequence && ( + + {sequence.error + ? "Sequence unavailable" + : sequence.jobSequence?.waitReason} + + )} +
    + ), +})); +vi.mock("@hooks/use-retry-jobs", () => ({ + useRetryJobs: ({ onSuccess }: { onSuccess: () => void }) => ({ + mutate: onSuccess, + }), +})); + +const job = jobFactory.pending().build({ metadata: { seq_key: "order" } }); + +beforeEach(() => { + featureState.enabled = true; + refreshState.intervalMs = 0; + getJob.mockReset().mockResolvedValue(job); + getSequence + .mockReset() + .mockResolvedValue(jobSequence({ jobId: job.id, waitReason: "up_next" })); + vi.spyOn(Route, "useParams").mockReturnValue({ jobId: job.id }); + vi.spyOn(Route, "useNavigate").mockReturnValue(vi.fn()); + vi.spyOn(Route, "useRouteContext").mockReturnValue({ + queryOptions: { queryFn: getJob, queryKey: getJobKey(job.id) }, + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function renderJob() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + , + ); + return client; +} + +test("loads a sequence job's status and refreshes it after retry", async () => { + renderJob(); + await screen.findByText("up_next"); + expect(getSequence).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + await waitFor(() => { + expect(getSequence).toHaveBeenCalledTimes(2); + }); +}); + +test("doesn't request a sequence from a backend without sequence queries", async () => { + featureState.enabled = false; + renderJob(); + await screen.findByText("Job details"); + expect(getSequence).not.toHaveBeenCalled(); +}); + +test("doesn't request a sequence for a job outside one", async () => { + getJob.mockResolvedValue({ ...job, metadata: {} }); + renderJob(); + await screen.findByText("Job details"); + expect(getSequence).not.toHaveBeenCalled(); +}); + +test("keeps the job page usable when the sequence fails to load", async () => { + getSequence.mockRejectedValue(new Error("Query timed out")); + renderJob(); + await screen.findByText("Sequence unavailable"); + expect(screen.getByText("Job details")).toBeInTheDocument(); +}); + +test("polls the sequence only while the job is pending", async () => { + refreshState.intervalMs = 2000; + const client = renderJob(); + await screen.findByText("up_next"); + const query = client + .getQueryCache() + .find({ queryKey: ["jobSequence", job.id.toString(), "pending"] }); + expect(query?.options).toMatchObject({ refetchInterval: 2000 }); +}); + +test("loads the sequence once for a job that isn't pending", async () => { + refreshState.intervalMs = 2000; + getJob.mockResolvedValue({ ...job, state: JobState.Cancelled }); + const client = renderJob(); + await screen.findByText("up_next"); + const query = client + .getQueryCache() + .find({ queryKey: ["jobSequence", job.id.toString(), "cancelled"] }); + expect(query?.options).toMatchObject({ + refetchInterval: false, + staleTime: Infinity, + }); +}); diff --git a/src/routes/sequences/$sequenceKey.tsx b/src/routes/sequences/$sequenceKey.tsx new file mode 100644 index 00000000..a7f6a20b --- /dev/null +++ b/src/routes/sequences/$sequenceKey.tsx @@ -0,0 +1,251 @@ +import SequenceDetail from "@components/SequenceDetail"; +import { useRefreshSetting } from "@contexts/RefreshSettings.hook"; +import { refreshQueryOptions } from "@contexts/RefreshSettings.query"; +import { useRetryJobs } from "@hooks/use-retry-jobs"; +import { deleteJobs, getJobKey } from "@services/jobs"; +import { + getSequenceInspection, + isActiveState, + listSequenceJobs, + type SequenceInspection, + sequenceInspectionKey, + type SequenceJobListCursor, + sequenceJobListKey, + type SequenceJobListPage, + sequenceKeyPrefix, +} from "@services/sequences"; +import { toastError, toastSuccess } from "@services/toast"; + +const getJobKeyPrefix = ["getJob"] as const; +const jobSequenceKeyPrefix = ["jobSequence"] as const; +import { + keepPreviousData, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { createFileRoute } from "@tanstack/react-router"; +import { useEffect, useRef, useState } from "react"; +import { z } from "zod"; + +const sequenceSearchSchema = z.object({ + // The job the operator came from, outlined in the list. + job: z.coerce.bigint().positive().optional(), +}); + +export const Route = createFileRoute("/sequences/$sequenceKey")({ + validateSearch: sequenceSearchSchema, + component: SequenceComponent, +}); + +// Jobs load in pages of this size, and the loaded range never grows past the +// API's maximum page, so polling refreshes it with a single request. +export const SEQUENCE_PAGE_SIZE = 100; +export const SEQUENCE_WINDOW_MAX = 1000; + +// SequenceWindow is the contiguous range of jobs loaded on the page. +type SequenceWindow = { cursor: SequenceJobListCursor; limit: number }; + +// initialSequenceWindow starts at the earliest job that explains the +// sequence's state: the halting job, the first blocking job, or the next +// pending job. The job the operator came from is the fallback, and a sequence +// with none of those shows its newest jobs. +export const initialSequenceWindow = ( + inspection: SequenceInspection, + selectedJobId?: bigint, +): SequenceWindow => { + const candidates = [ + inspection.haltedBy?.id, + inspection.blockers[0]?.id, + inspection.head?.id, + ].filter((id): id is bigint => id !== undefined); + const anchor = candidates.length + ? candidates.reduce((min, id) => (id < min ? id : min)) + : selectedJobId; + return anchor === undefined + ? { cursor: { newest: true }, limit: SEQUENCE_PAGE_SIZE } + : { cursor: { afterId: anchor - 1n }, limit: SEQUENCE_PAGE_SIZE }; +}; + +// A sequence changes on its own only while a job in it is active, or while +// its next pending job isn't held by a halting job. Otherwise it changes only +// through actions like a retry, which refresh the page themselves. +export const sequenceCanChange = (inspection?: SequenceInspection) => + !inspection || + inspection.blockers.some((job) => isActiveState(job.state)) || + (!!inspection.head && !inspection.haltedBy); + +export function SequenceComponent() { + const { sequenceKey } = Route.useParams(); + const { job: selectedJobId } = Route.useSearch(); + const queryClient = useQueryClient(); + const refreshSettings = useRefreshSetting(); + + const [range, setRange] = useState(null); + const [loadingMore, setLoadingMore] = useState<"earlier" | "later" | null>( + null, + ); + const [fetchingPage, setFetchingPage] = useState(false); + + const inspectionQuery = useQuery({ + queryFn: getSequenceInspection, + queryKey: sequenceInspectionKey(sequenceKey), + ...refreshQueryOptions(refreshSettings.intervalMs), + // Poll only while the sequence can change on its own. + refetchInterval: (query) => + refreshSettings.intervalMs > 0 && sequenceCanChange(query.state.data) + ? refreshSettings.intervalMs + : false, + }); + const inspection = inspectionQuery.data; + + // Pick the starting range once the first inspection arrives. Later + // refreshes keep the range the operator is looking at. + if (inspection && !range) { + setRange(initialSequenceWindow(inspection, selectedJobId)); + } + + const jobsQuery = useQuery({ + enabled: !!range, + placeholderData: keepPreviousData, + queryFn: async ({ signal }): Promise => { + if (!range) throw new Error("No range of jobs selected."); + return listSequenceJobs(sequenceKey, range.cursor, range.limit, signal); + }, + // Without a range the query is disabled, and its key is never fetched. + queryKey: sequenceJobListKey( + sequenceKey, + range?.cursor ?? { newest: true }, + range?.limit ?? 0, + ), + }); + const jobs = jobsQuery.data?.jobs ?? []; + // A button stays busy while its page loads: first the earlier page itself, + // then the wider range that replaces the shown jobs. + const busy = (direction: "earlier" | "later") => + loadingMore === direction && (fetchingPage || jobsQuery.isPlaceholderData); + + // The list refreshes along with the inspection: whenever a poll or an action + // sees the sequence change (like a job finishing), reload the visible range. + const fingerprint = inspection ? inspectionFingerprint(inspection) : null; + const lastFingerprint = useRef(fingerprint); + useEffect(() => { + if (fingerprint === lastFingerprint.current) return; + const changed = lastFingerprint.current !== null; + lastFingerprint.current = fingerprint; + if (changed) { + void queryClient.invalidateQueries({ + queryKey: [...sequenceKeyPrefix(sequenceKey), "jobs"], + }); + } + }, [fingerprint, queryClient, sequenceKey]); + + // Showing later jobs raises the range's limit, and slides it forward once + // it's at the maximum. The newest jobs have nothing later to show. + const showLater = () => { + if (!range || !("afterId" in range.cursor)) return; + setLoadingMore("later"); + if (range.limit + SEQUENCE_PAGE_SIZE <= SEQUENCE_WINDOW_MAX) { + setRange({ ...range, limit: range.limit + SEQUENCE_PAGE_SIZE }); + return; + } + const dropped = jobs[SEQUENCE_PAGE_SIZE - 1]; + if (dropped) { + setRange({ cursor: { afterId: dropped.id }, limit: range.limit }); + } + }; + + // Showing earlier jobs loads the page before the shown jobs to find where + // the wider range starts. Past the maximum, the latest jobs drop from view. + const showEarlier = async () => { + const first = jobs[0]; + if (!first || !range) return; + setLoadingMore("earlier"); + setFetchingPage(true); + try { + const cursor = { beforeId: first.id }; + const page = await queryClient.fetchQuery({ + queryFn: ({ signal }) => + listSequenceJobs(sequenceKey, cursor, SEQUENCE_PAGE_SIZE, signal), + queryKey: sequenceJobListKey(sequenceKey, cursor, SEQUENCE_PAGE_SIZE), + }); + const start = page.jobs[0]?.id ?? first.id; + setRange({ + cursor: { afterId: start - 1n }, + limit: Math.min(SEQUENCE_WINDOW_MAX, jobs.length + page.jobs.length), + }); + } catch (error) { + toastError({ + message: "Couldn't load earlier jobs", + subtext: error instanceof Error ? error.message : undefined, + }); + } finally { + setFetchingPage(false); + } + }; + + const jumpToJob = (jobId: bigint) => + setRange({ cursor: { afterId: jobId - 1n }, limit: SEQUENCE_PAGE_SIZE }); + + // Retrying or deleting a halting job changes the whole sequence, so cached + // job pages for any job in it are stale too. + const refreshSequence = () => + Promise.all([ + queryClient.invalidateQueries({ + queryKey: sequenceKeyPrefix(sequenceKey), + }), + queryClient.invalidateQueries({ queryKey: getJobKeyPrefix }), + queryClient.invalidateQueries({ queryKey: jobSequenceKeyPrefix }), + ]); + + const retryMutation = useRetryJobs({ + onSuccess: refreshSequence, + successMessage: "Job enqueued for retry", + }); + + const deleteMutation = useMutation({ + mutationFn: async (jobId, context) => deleteJobs({ ids: [jobId] }, context), + onError: (error) => { + toastError({ message: "Job delete failed", subtext: error.message }); + }, + onSuccess: (_data, jobId) => { + toastSuccess({ duration: 2000, message: "Job deleted" }); + queryClient.removeQueries({ queryKey: getJobKey(jobId) }); + return refreshSequence(); + }, + }); + + return ( + { + if (action === "retry") { + retryMutation.mutate([jobId]); + } else { + deleteMutation.mutate(jobId); + } + }} + onShowEarlier={() => void showEarlier()} + onShowLater={showLater} + selectedJobId={selectedJobId} + sequenceKey={sequenceKey} + showingEarlier={busy("earlier")} + showingLater={busy("later")} + /> + ); +} + +const inspectionFingerprint = (inspection: SequenceInspection) => + JSON.stringify([ + inspection.haltedBy?.id.toString(), + inspection.head?.id.toString(), + inspection.blockers.map((job) => `${job.id}:${job.state}`), + inspection.pendingCount, + ]); diff --git a/src/routes/sequences/sequence.test.tsx b/src/routes/sequences/sequence.test.tsx new file mode 100644 index 00000000..0e4c5fdf --- /dev/null +++ b/src/routes/sequences/sequence.test.tsx @@ -0,0 +1,235 @@ +import type { SequenceDetailProps } from "@components/SequenceDetail"; + +import { + initialSequenceWindow, + Route, + SEQUENCE_PAGE_SIZE, + sequenceCanChange, + SequenceComponent, +} from "@routes/sequences/$sequenceKey"; +import { JobState } from "@services/types"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { + haltedSequenceFixture, + sequenceInspection, + sequenceJob, + sequenceKeyFixture, + sequenceListItem, +} from "@test/factories/sequence"; +import { act, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const { getInspection, listJobs, page, refreshState, retry } = vi.hoisted( + () => ({ + getInspection: vi.fn(), + listJobs: vi.fn(), + page: { props: undefined as SequenceDetailProps | undefined }, + refreshState: { intervalMs: 0 }, + retry: vi.fn(), + }), +); + +vi.mock("@contexts/RefreshSettings.hook", () => ({ + useRefreshSetting: () => ({ intervalMs: refreshState.intervalMs }), +})); +vi.mock("@services/sequences", async (original) => ({ + ...(await original()), + getSequenceInspection: getInspection, + listSequenceJobs: listJobs, +})); +vi.mock("@components/SequenceDetail", () => ({ + default: (props: SequenceDetailProps) => { + page.props = props; + return
    {props.jobs.map((job) => `#${job.id}`).join(" ")}
    ; + }, +})); +vi.mock("@hooks/use-retry-jobs", () => ({ + useRetryJobs: ({ onSuccess }: { onSuccess: () => void }) => ({ + mutate: (ids: bigint[]) => { + retry(ids); + return onSuccess(); + }, + }), +})); + +const props = () => { + if (!page.props) throw new Error("Page not rendered"); + return page.props; +}; + +beforeEach(() => { + page.props = undefined; + refreshState.intervalMs = 0; + getInspection.mockReset().mockResolvedValue(haltedSequenceFixture.inspection); + listJobs.mockReset().mockResolvedValue({ + hasLater: true, + hasEarlier: true, + jobs: haltedSequenceFixture.jobs, + }); + vi.spyOn(Route, "useParams").mockReturnValue({ + sequenceKey: sequenceKeyFixture, + }); + vi.spyOn(Route, "useSearch").mockReturnValue({ job: 7010n }); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function renderSequence() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + , + ); + return client; +} + +describe("initialSequenceWindow", () => { + test("starts at the halting job", () => { + expect( + initialSequenceWindow(haltedSequenceFixture.inspection, 7010n), + ).toEqual({ + cursor: { afterId: 7000n }, + limit: SEQUENCE_PAGE_SIZE, + }); + }); + + test("starts at the earliest blocking or pending job", () => { + const inspection = sequenceInspection({ + blockers: [sequenceJob(7002n, JobState.Running)], + head: sequenceJob(6999n, JobState.Pending), + }); + expect(initialSequenceWindow(inspection).cursor).toEqual({ + afterId: 6998n, + }); + }); + + test("lists from the first job when it has ID 1", () => { + const inspection = sequenceInspection({ + haltedBy: sequenceJob(1n, JobState.Cancelled), + }); + expect(initialSequenceWindow(inspection).cursor).toEqual({ afterId: 0n }); + }); + + test("falls back to the selected job, then the newest jobs", () => { + expect(initialSequenceWindow(sequenceInspection(), 6996n).cursor).toEqual({ + afterId: 6995n, + }); + expect(initialSequenceWindow(sequenceInspection()).cursor).toEqual({ + newest: true, + }); + }); +}); + +test("sequenceCanChange only while a job is active or the head can run", () => { + expect(sequenceCanChange(haltedSequenceFixture.inspection)).toBe(false); + expect(sequenceCanChange(sequenceInspection())).toBe(false); + expect( + sequenceCanChange( + sequenceInspection({ head: sequenceJob(7003n, JobState.Pending) }), + ), + ).toBe(true); + expect( + sequenceCanChange( + sequenceInspection({ + blockers: [sequenceJob(7002n, JobState.Retryable)], + haltedBy: sequenceJob(7001n, JobState.Cancelled), + }), + ), + ).toBe(true); +}); + +test("loads jobs from the halting job and outlines the selected job", async () => { + renderSequence(); + await screen.findByText("#7001 #7003 #7010 #7022"); + expect(listJobs).toHaveBeenCalledWith( + sequenceKeyFixture, + { afterId: 7000n }, + SEQUENCE_PAGE_SIZE, + expect.anything(), + ); + expect(props().selectedJobId).toBe(7010n); + expect(props().hasEarlier).toBe(true); +}); + +test("shows earlier jobs by widening the loaded range", async () => { + renderSequence(); + await screen.findByText("#7001 #7003 #7010 #7022"); + const earlier = [ + sequenceListItem(6990n, JobState.Completed, 900, "placed"), + sequenceListItem(6996n, JobState.Completed, 600, "paid"), + ]; + listJobs.mockImplementation((_key, cursor: object) => + Promise.resolve( + "beforeId" in cursor + ? { hasLater: true, hasEarlier: false, jobs: earlier } + : { + hasLater: true, + hasEarlier: false, + jobs: [...earlier, ...haltedSequenceFixture.jobs], + }, + ), + ); + + await act(async () => props().onShowEarlier()); + await screen.findByText("#6990 #6996 #7001 #7003 #7010 #7022"); + expect(listJobs).toHaveBeenCalledWith( + sequenceKeyFixture, + { beforeId: 7001n }, + SEQUENCE_PAGE_SIZE, + expect.anything(), + ); + expect(listJobs).toHaveBeenLastCalledWith( + sequenceKeyFixture, + { afterId: 6989n }, + 6, + expect.anything(), + ); +}); + +test("shows later jobs by loading a longer range", async () => { + renderSequence(); + await screen.findByText("#7001 #7003 #7010 #7022"); + act(() => props().onShowLater()); + await waitFor(() => { + expect(listJobs).toHaveBeenLastCalledWith( + sequenceKeyFixture, + { afterId: 7000n }, + SEQUENCE_PAGE_SIZE * 2, + expect.anything(), + ); + }); +}); + +test("polls only while the sequence can change", async () => { + refreshState.intervalMs = 2000; + const client = renderSequence(); + await screen.findByText("#7001 #7003 #7010 #7022"); + const query = client + .getQueryCache() + .find({ queryKey: ["sequence", sequenceKeyFixture, "inspection"] }); + if (!query) throw new Error("Expected an inspection query"); + const interval = query.observers[0]?.options.refetchInterval; + if (typeof interval !== "function") throw new Error("Expected a function"); + expect(interval(query)).toBe(false); + + query.setData( + sequenceInspection({ blockers: [sequenceJob(7002n, JobState.Running)] }), + ); + expect(interval(query)).toBe(2000); +}); + +test("retrying the halting job reloads the sequence", async () => { + renderSequence(); + await screen.findByText("#7001 #7003 #7010 #7022"); + expect(getInspection).toHaveBeenCalledTimes(1); + act(() => props().onRemoveHalt(7001n, "retry")); + expect(retry).toHaveBeenCalledWith([7001n]); + await waitFor(() => { + expect(getInspection).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/services/features.test.ts b/src/services/features.test.ts index a33d9e70..26eaa864 100644 --- a/src/services/features.test.ts +++ b/src/services/features.test.ts @@ -22,6 +22,7 @@ describe("apiFeaturesToFeatures", () => { hasSequenceTable: false, jobListHideArgsByDefault: true, producerQueries: true, + sequenceQueries: false, workflowQueries: true, }; @@ -46,6 +47,7 @@ describe("apiFeaturesToFeatures", () => { hasSequenceTable: false, jobListHideArgsByDefault: false, producerQueries: false, + sequenceQueries: false, workflowQueries: false, }; diff --git a/src/services/features.ts b/src/services/features.ts index 20254d8f..4da4c8eb 100644 --- a/src/services/features.ts +++ b/src/services/features.ts @@ -19,6 +19,7 @@ const KNOWN_EXTENSIONS = [ "dead_letter_jobs", "durable_periodic_jobs", "producer_queries", + "sequence_queries", "workflow_queries", "has_producer_table", "has_sequence_table", diff --git a/src/services/sequences.test.ts b/src/services/sequences.test.ts new file mode 100644 index 00000000..76f4bf59 --- /dev/null +++ b/src/services/sequences.test.ts @@ -0,0 +1,127 @@ +import { JobState } from "@services/types"; +import { describe, expect, test } from "vitest"; + +import { + formatPendingCount, + jobSequenceFromAPI, + sequenceJobListItemFromAPI, + sequenceKeyFromMetadata, + sequenceWaitSummary, +} from "./sequences"; + +describe("jobSequenceFromAPI", () => { + test("preserves job IDs beyond the JavaScript integer range", () => { + const blocker = { + id: "1970670598291982290", + kind: "order", + queue: "default", + scheduled_at: "2026-11-13T12:00:00Z", + state: JobState.Cancelled, + }; + const jobSequence = jobSequenceFromAPI({ + halting: false, + job_id: "1970670598291982291", + job_state: JobState.Pending, + sequence: { + blockers: [blocker], + halted_by: blocker, + head: null, + key: "order", + pending_count: null, + }, + wait_reason: "halted", + waiting_on: blocker, + }); + expect(jobSequence.jobId).toBe(1970670598291982291n); + expect(jobSequence.waitingOn?.id).toBe(1970670598291982290n); + expect(jobSequence.sequence.haltedBy?.scheduledAt).toEqual( + new Date("2026-11-13T12:00:00Z"), + ); + expect(jobSequence.sequence.head).toBeNull(); + }); +}); + +test("sequenceJobListItemFromAPI converts IDs and optional times", () => { + const item = sequenceJobListItemFromAPI({ + args: '{"order_id":1}', + attempt: 0, + attempted_at: null, + created_at: "2026-11-13T12:00:00Z", + finalized_at: null, + id: "9007199254740993", + kind: "order", + max_attempts: 25, + priority: 1, + queue: "default", + scheduled_at: "2026-11-13T12:00:00Z", + state: JobState.Pending, + }); + expect(item.id).toBe(9007199254740993n); + expect(item.argsRaw).toBe('{"order_id":1}'); + expect(item.attemptedAt).toBeUndefined(); + expect(item.finalizedAt).toBeUndefined(); +}); + +test("sequenceKeyFromMetadata only accepts a nonempty string key", () => { + expect(sequenceKeyFromMetadata({ seq_key: "order" })).toBe("order"); + for (const metadata of [ + {}, + { seq_key: null }, + { seq_key: "" }, + { seq_key: 123 }, + { workflow_id: "workflow" }, + ]) { + expect(sequenceKeyFromMetadata(metadata)).toBeUndefined(); + } +}); + +test("formatPendingCount marks a capped count", () => { + expect(formatPendingCount({ count: 2341, more: false })).toBe("2,341"); + expect(formatPendingCount({ count: 1000, more: true })).toBe("1,000+"); +}); + +describe("sequenceWaitSummary", () => { + const status = (waitReason: "halted" | "up_next" | "waiting_on_active") => ({ + error: false, + jobSequence: { + halting: false, + jobId: 1n, + jobState: JobState.Pending, + sequence: { + blockers: [], + haltedBy: null, + head: null, + key: "k", + pendingCount: null, + }, + waitingOn: null, + waitReason, + }, + key: "k", + }); + const pending = { state: JobState.Pending }; + + test("labels and tints a pending job's wait", () => { + expect(sequenceWaitSummary(pending, status("halted"))).toEqual({ + label: "Sequence halted", + tone: "halted", + }); + expect(sequenceWaitSummary(pending, status("waiting_on_active"))).toEqual({ + label: "Waiting on sequence", + tone: "waiting", + }); + expect(sequenceWaitSummary(pending, status("up_next"))?.tone).toBe( + "neutral", + ); + }); + + test("says nothing for other jobs, failures, or missing status", () => { + expect( + sequenceWaitSummary({ state: JobState.Completed }, status("halted")), + ).toBeUndefined(); + expect( + sequenceWaitSummary(pending, { ...status("halted"), error: true }), + ).toBeUndefined(); + expect(sequenceWaitSummary(pending)).toBeUndefined(); + }); +}); diff --git a/src/services/sequences.ts b/src/services/sequences.ts new file mode 100644 index 00000000..8e0b16d1 --- /dev/null +++ b/src/services/sequences.ts @@ -0,0 +1,307 @@ +import type { QueryFunction } from "@tanstack/react-query"; + +import { API } from "@utils/api"; + +import { JobState } from "./types"; + +// JobSequence explains where one job stands in its sequence. +export type JobSequence = { + // True when the job is cancelled or discarded without letting its sequence + // continue, and pending jobs wait behind it. + halting: boolean; + jobId: bigint; + jobState: JobState; + sequence: SequenceInspection; + // The job that explains `waitReason`: the halting job, the active job, or + // the earlier pending job. Null for `none` and `up_next`. + waitingOn: null | SequenceJob; + waitReason: SequenceWaitReason; +}; + +// JobSequenceStatus is what the job detail page knows about the sequence a job +// belongs to. The key comes from job metadata and is always available, while +// the sequence snapshot is loaded separately and may be missing or failed. +export type JobSequenceStatus = { + error: boolean; + jobSequence?: JobSequence; + key: string; +}; + +export type SequenceInspection = { + // The earliest blocking job in each state, in ID order. + blockers: SequenceJob[]; + // The earliest cancelled or discarded job stopping the sequence. + haltedBy: null | SequenceJob; + // The sequence's earliest pending job. + head: null | SequenceJob; + key: string; + // Pending jobs in the sequence, capped by the server. Null when the server + // didn't count them. + pendingCount: null | SequencePendingCount; +}; + +export type SequenceJob = { + id: bigint; + kind: string; + queue: string; + scheduledAt: Date; + state: JobState; +}; + +export type SequenceJobListItem = { + argsRaw: string; + attempt: number; + attemptedAt?: Date; + createdAt: Date; + finalizedAt?: Date; + id: bigint; + kind: string; + maxAttempts: number; + priority: number; + queue: string; + scheduledAt: Date; + state: JobState; +}; + +export type SequenceJobListPage = { + hasEarlier: boolean; + hasLater: boolean; + jobs: SequenceJobListItem[]; +}; + +export type SequencePendingCount = { + count: number; + // True when the sequence has more than `count` pending jobs. + more: boolean; +}; + +export type SequenceWaitReason = + | "halted" + | "none" + | "up_next" + | "waiting_behind_pending" + | "waiting_on_active"; + +type JobSequenceFromAPI = { + halting: boolean; + job_id: string; + job_state: JobState; + sequence: SequenceInspectionFromAPI; + wait_reason: SequenceWaitReason; + waiting_on: null | SequenceJobFromAPI; +}; + +type SequenceInspectionFromAPI = { + blockers: SequenceJobFromAPI[]; + halted_by: null | SequenceJobFromAPI; + head: null | SequenceJobFromAPI; + key: string; + pending_count: null | SequencePendingCount; +}; + +type SequenceJobFromAPI = { + id: string; + kind: string; + queue: string; + scheduled_at: string; + state: JobState; +}; + +type SequenceJobListItemFromAPI = { + args: string; + attempt: number; + attempted_at: null | string; + created_at: string; + finalized_at: null | string; + id: string; + kind: string; + max_attempts: number; + priority: number; + queue: string; + scheduled_at: string; + state: JobState; +}; + +type SequenceJobListPageFromAPI = { + data: SequenceJobListItemFromAPI[]; + has_earlier: boolean; + has_later: boolean; +}; + +// Cancelled and discarded jobs halt their sequence unless they were inserted +// with the matching continuation option, which the API accounts for. +export const isHaltingState = (state: JobState) => + state === JobState.Cancelled || state === JobState.Discarded; + +// Jobs in these states block the rest of their sequence from anywhere in it, +// and change on their own, so a sequence with one is worth polling. +export const isActiveState = (state: JobState) => + state === JobState.Available || + state === JobState.Retryable || + state === JobState.Running || + state === JobState.Scheduled; + +export const sequenceKeyFromMetadata = ( + metadata: object, +): string | undefined => { + if ( + "seq_key" in metadata && + typeof metadata.seq_key === "string" && + metadata.seq_key.length > 0 + ) { + return metadata.seq_key; + } + return undefined; +}; + +export type SequenceWaitTone = "halted" | "neutral" | "waiting"; + +// sequenceWaitSummary is a short label and tone for a pending job's wait +// step on the timeline, so it says more than "Pending". +export const sequenceWaitSummary = ( + job: { state: JobState }, + status?: JobSequenceStatus, +): { label: string; tone: SequenceWaitTone } | undefined => { + if (job.state !== JobState.Pending || status?.error || !status?.jobSequence) { + return undefined; + } + switch (status.jobSequence.waitReason) { + case "halted": + return { label: "Sequence halted", tone: "halted" }; + case "none": + return undefined; + case "up_next": + return { label: "Up next in sequence", tone: "neutral" }; + case "waiting_behind_pending": + case "waiting_on_active": + return { label: "Waiting on sequence", tone: "waiting" }; + } +}; + +// formatPendingCount renders a capped count, like "1,000+". +export const formatPendingCount = (count: SequencePendingCount): string => + `${count.count.toLocaleString()}${count.more ? "+" : ""}`; + +const sequenceJobFromAPI = (job: SequenceJobFromAPI): SequenceJob => ({ + id: BigInt(job.id), + kind: job.kind, + queue: job.queue, + scheduledAt: new Date(job.scheduled_at), + state: job.state, +}); + +const nullableSequenceJobFromAPI = ( + job: null | SequenceJobFromAPI, +): null | SequenceJob => (job ? sequenceJobFromAPI(job) : null); + +export const sequenceInspectionFromAPI = ( + inspection: SequenceInspectionFromAPI, +): SequenceInspection => ({ + blockers: inspection.blockers.map(sequenceJobFromAPI), + haltedBy: nullableSequenceJobFromAPI(inspection.halted_by), + head: nullableSequenceJobFromAPI(inspection.head), + key: inspection.key, + pendingCount: inspection.pending_count, +}); + +export const jobSequenceFromAPI = ( + jobSequence: JobSequenceFromAPI, +): JobSequence => ({ + halting: jobSequence.halting, + jobId: BigInt(jobSequence.job_id), + jobState: jobSequence.job_state, + sequence: sequenceInspectionFromAPI(jobSequence.sequence), + waitingOn: nullableSequenceJobFromAPI(jobSequence.waiting_on), + waitReason: jobSequence.wait_reason, +}); + +export const sequenceJobListItemFromAPI = ( + job: SequenceJobListItemFromAPI, +): SequenceJobListItem => ({ + argsRaw: job.args, + attempt: job.attempt, + attemptedAt: job.attempted_at ? new Date(job.attempted_at) : undefined, + createdAt: new Date(job.created_at), + finalizedAt: job.finalized_at ? new Date(job.finalized_at) : undefined, + id: BigInt(job.id), + kind: job.kind, + maxAttempts: job.max_attempts, + priority: job.priority, + queue: job.queue, + scheduledAt: new Date(job.scheduled_at), + state: job.state, +}); + +// The job's state is part of the key so a state change (like the job being +// retried, or leaving `pending`) loads a fresh snapshot even when polling is +// off. Invalidate every state with `jobSequenceKey(jobId)`. +export const jobSequenceKey = (jobId: bigint, jobState?: JobState) => + jobState + ? (["jobSequence", jobId.toString(), jobState] as const) + : (["jobSequence", jobId.toString()] as const); + +export const getJobSequence: QueryFunction< + JobSequence, + ReturnType +> = async ({ queryKey: [, jobId], signal }) => + API.get( + { path: `/pro/jobs/${jobId}/sequence` }, + { signal }, + ).then(jobSequenceFromAPI); + +// Matches every query for one sequence, for invalidation after an action. +export const sequenceKeyPrefix = (key: string) => ["sequence", key] as const; + +export const sequenceInspectionKey = (key: string) => + ["sequence", key, "inspection"] as const; + +export const getSequenceInspection: QueryFunction< + SequenceInspection, + ReturnType +> = async ({ queryKey: [, key], signal }) => + API.get( + { path: "/pro/sequence", query: new URLSearchParams({ key }) }, + { signal }, + ).then(sequenceInspectionFromAPI); + +// SequenceJobListCursor selects a page of a sequence's jobs. Without a cursor +// the page holds the newest jobs. +export type SequenceJobListCursor = + { afterId: bigint } | { beforeId: bigint } | { newest: true }; + +export const sequenceJobListKey = ( + key: string, + cursor: SequenceJobListCursor, + limit: number, +) => + [ + "sequence", + key, + "jobs", + "afterId" in cursor + ? { afterId: cursor.afterId.toString() } + : "beforeId" in cursor + ? { beforeId: cursor.beforeId.toString() } + : { newest: true }, + limit, + ] as const; + +export const listSequenceJobs = async ( + key: string, + cursor: SequenceJobListCursor, + limit: number, + signal?: AbortSignal, +): Promise => { + const query = new URLSearchParams({ key, limit: limit.toString() }); + if ("afterId" in cursor) query.set("after_id", cursor.afterId.toString()); + if ("beforeId" in cursor) query.set("before_id", cursor.beforeId.toString()); + const page = await API.get( + { path: "/pro/sequence/jobs", query }, + { signal }, + ); + return { + hasEarlier: page.has_earlier, + hasLater: page.has_later, + jobs: page.data.map(sequenceJobListItemFromAPI), + }; +}; diff --git a/src/test/factories/sequence.ts b/src/test/factories/sequence.ts new file mode 100644 index 00000000..dcd3fd22 --- /dev/null +++ b/src/test/factories/sequence.ts @@ -0,0 +1,96 @@ +import type { + JobSequence, + SequenceInspection, + SequenceJob, + SequenceJobListItem, +} from "@services/sequences"; + +import { JobState } from "@services/types"; + +// Sequence keys are usually hashes, like this base64 SHA-256 digest. +export const sequenceKeyFixture = + "kM2xQ7pR9tVb3nY8wZ1aC4dF6gH0jL5sE7uI2oP9qT4="; + +const minutesFromNow = (minutes: number) => + new Date(Date.now() + minutes * 60_000); + +export const sequenceJob = ( + id: bigint, + state: JobState, + overrides: Partial = {}, +): SequenceJob => ({ + id, + kind: "update_order_status", + queue: "orders", + scheduledAt: minutesFromNow(state === JobState.Retryable ? 4 : -5), + state, + ...overrides, +}); + +export const sequenceInspection = ( + overrides: Partial = {}, +): SequenceInspection => ({ + blockers: [], + haltedBy: null, + head: null, + key: sequenceKeyFixture, + pendingCount: null, + ...overrides, +}); + +export const jobSequence = ( + overrides: Partial = {}, +): JobSequence => ({ + halting: false, + jobId: 7003n, + jobState: JobState.Pending, + sequence: sequenceInspection(), + waitingOn: null, + waitReason: "none", + ...overrides, +}); + +export const sequenceListItem = ( + id: bigint, + state: JobState, + minutesAgo: number, + status: string, +): SequenceJobListItem => { + const at = minutesFromNow(-minutesAgo); + const finalized = [ + JobState.Cancelled, + JobState.Completed, + JobState.Discarded, + ].includes(state); + return { + argsRaw: JSON.stringify({ order_id: 48213, status }), + attempt: state === JobState.Pending ? 0 : 1, + attemptedAt: state === JobState.Pending ? undefined : at, + createdAt: minutesFromNow(-minutesAgo - 1), + finalizedAt: finalized ? at : undefined, + id, + kind: id === 6990n ? "process_order" : "update_order_status", + maxAttempts: 25, + priority: 1, + queue: "orders", + scheduledAt: state === JobState.Retryable ? minutesFromNow(4) : at, + state, + }; +}; + +// A sequence halted by a cancelled job, with completed jobs before it and +// pending jobs waiting behind it. +export const haltedSequenceFixture = { + inspection: sequenceInspection({ + blockers: [sequenceJob(7001n, JobState.Cancelled)], + haltedBy: sequenceJob(7001n, JobState.Cancelled), + head: sequenceJob(7003n, JobState.Pending), + pendingCount: { count: 2341, more: false }, + }), + jobs: [ + sequenceListItem(7001n, JobState.Cancelled, 420, "packed"), + sequenceListItem(7003n, JobState.Pending, 300, "shipped"), + sequenceListItem(7010n, JobState.Pending, 120, "delivered"), + sequenceListItem(7022n, JobState.Pending, 30, "archived"), + ], +}; diff --git a/src/test/utils/features.ts b/src/test/utils/features.ts index aed5d078..5ad7d0ed 100644 --- a/src/test/utils/features.ts +++ b/src/test/utils/features.ts @@ -9,6 +9,7 @@ export const createFeatures = ( hasSequenceTable: false, jobListHideArgsByDefault: false, producerQueries: false, + sequenceQueries: false, workflowQueries: false, ...overrides, }); diff --git a/src/test/utils/router.tsx b/src/test/utils/router.tsx new file mode 100644 index 00000000..cf6b8b22 --- /dev/null +++ b/src/test/utils/router.tsx @@ -0,0 +1,21 @@ +import type { ReactNode } from "react"; + +import { + createMemoryHistory, + createRootRoute, + createRouter, + RouterProvider, +} from "@tanstack/react-router"; +import { act, render } from "@testing-library/react"; + +// renderWithRouter renders a component that uses router links. Links build +// their href from `to` and params, so the tree doesn't need their routes. +export const renderWithRouter = async (ui: () => ReactNode) => { + const rootRoute = createRootRoute({ component: ui }); + const router = createRouter({ + history: createMemoryHistory({ initialEntries: ["/"] }), + routeTree: rootRoute, + }); + await act(() => router.load()); + return render(); +}; diff --git a/src/test/utils/text.ts b/src/test/utils/text.ts new file mode 100644 index 00000000..ca2b7e9b --- /dev/null +++ b/src/test/utils/text.ts @@ -0,0 +1,8 @@ +// visibleText is an element's text without SVG titles, which give icons an +// accessible name but aren't shown. +export const visibleText = (element: Element | null | undefined): string => { + if (!element) return ""; + const clone = element.cloneNode(true) as Element; + clone.querySelectorAll("title").forEach((title) => title.remove()); + return clone.textContent ?? ""; +};