Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ trader data verify ... # check canonical bars against the raw they wer
trader data audit ... # one integrity report for a range (text or JSON)
trader data migrate ... # move an old-layout raw archive to the current layout (dry run unless --apply)
trader backtest ... # backtest canonical bars from the data store
trader serve # read-only REST API over the canonical bars (docs/rest-api.md)
trader serve # read-only REST API over the canonical bars (docs/rest-api.md) and the loopback strategy service (ADR-094)
```

`trader data download` fetches bid/ask candles for a UTC range into month
Expand Down
87 changes: 85 additions & 2 deletions cmd/trader/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"github.com/rustyeddy/trader/internal/store"
"github.com/rustyeddy/trader/rest"
"github.com/rustyeddy/trader/service"
"github.com/rustyeddy/trader/strategy/proto"
strategyserver "github.com/rustyeddy/trader/strategy/server"
)

// envAPIToken is where the bearer token the API requires comes from (like the
Expand All @@ -25,9 +27,17 @@ const envAPIToken = "TRADER_API_TOKEN"

const defaultListen = "127.0.0.1:8080"

// defaultStrategyListen is where the strategy service listens unless told
// otherwise: loopback only, like the API. strategyOff turns it off.
const (
defaultStrategyListen = "127.0.0.1:8081"
strategyOff = "off"
)

type serveFlags struct {
config, dataRoot string
listen string
strategyListen string
timeout time.Duration
}

Expand All @@ -48,7 +58,13 @@ environment, in which case every request must carry "Authorization: Bearer <toke
There is no TLS: put it behind a proxy if it leaves the machine. The data root is
--data-root, then ` + envDataRoot + `, then marketdata.root in ` + defaultConfigPath + `, then
the default ` + store.DefaultRoot + `. It runs until interrupted and then finishes the
requests in flight.`,
requests in flight.

It also starts the strategy service, the loopback TCP protocol external strategies
connect to (ADR-094), on ` + defaultStrategyListen + ` unless --strategy-listen names another
loopback address; "` + strategyOff + `" turns it off. That service has no TLS or token, so an
address that is not loopback is refused. Until the engine is attached to it every
registration is refused with an explanation, and no run starts.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return a.runServe(cmd, f)
Expand All @@ -58,6 +74,7 @@ requests in flight.`,
fl.StringVar(&f.config, "config", defaultConfigPath, "config file (a missing default file is fine when flags or the environment supply the values)")
fl.StringVar(&f.dataRoot, "data-root", "", "data root (default: "+envDataRoot+", then marketdata.root in the config file, then "+store.DefaultRoot+")")
fl.StringVar(&f.listen, "listen", defaultListen, "address to listen on, host:port (a non-loopback host needs "+envAPIToken+")")
fl.StringVar(&f.strategyListen, "strategy-listen", defaultStrategyListen, "loopback address of the strategy service, host:port, or \""+strategyOff+"\" (a non-loopback host is refused)")
fl.DurationVar(&f.timeout, "request-timeout", rest.DefaultTimeout, "longest one request may take")
return cmd
}
Expand Down Expand Up @@ -89,6 +106,15 @@ func (a *app) runServe(cmd *cobra.Command, f serveFlags) error {
if f.timeout <= 0 {
return errors.New("--request-timeout must be positive")
}
if f.strategyListen != strategyOff {
slocal, err := strategyserver.IsLoopbackAddr(f.strategyListen)
if err != nil {
return fmt.Errorf("--strategy-listen: %w", err)
}
if !slocal {
return fmt.Errorf("--strategy-listen %q is not a loopback address: the strategy protocol has no TLS or token in v1", f.strategyListen)
}
}
cfg, err := loadConfig(f.config, cmd.Flags().Changed("config"))
if err != nil {
return err
Expand All @@ -101,6 +127,14 @@ func (a *app) runServe(cmd *cobra.Command, f serveFlags) error {
return fmt.Errorf("listen: %w", err)
}

var sln net.Listener
if f.strategyListen != strategyOff {
if sln, err = strategyserver.Listen(f.strategyListen); err != nil {
_ = ln.Close()
return fmt.Errorf("--strategy-listen: %w", err)
}
}

ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt)
defer stop()
log := logging.WithComponent(slog.Default(), logging.ComponentMarketData)
Expand Down Expand Up @@ -129,20 +163,69 @@ func (a *app) runServe(cmd *cobra.Command, f serveFlags) error {
return fmt.Errorf("write: %w", werr)
}

// The strategy service: nil channel when it is off, so its case never fires.
var sdone chan struct{}
var serr error
if sln != nil {
ssrv, nerr := strategyserver.New(strategyserver.Config{
Admit: admitNoEngine,
Handle: func(context.Context, *strategyserver.Session) error { return errNoEngine },
Logger: logging.WithComponent(slog.Default(), logging.ComponentStrategy),
})
if nerr != nil {
_ = sln.Close()
_ = ln.Close()
return nerr
}
if _, werr := fmt.Fprintf(cmd.OutOrStdout(), "strategy protocol on tcp://%s (loopback only; runs are refused until the engine is attached)\n", sln.Addr()); werr != nil {
_ = sln.Close()
_ = ln.Close()
return fmt.Errorf("write: %w", werr)
}
sdone = make(chan struct{})
go func() { serr = ssrv.Serve(ctx, sln); close(sdone) }()
} else if _, werr := fmt.Fprintln(cmd.OutOrStdout(), "strategy protocol off"); werr != nil {
_ = ln.Close()
return fmt.Errorf("write: %w", werr)
}
stopStrategy := func() {
stop()
if sdone != nil {
<-sdone
}
}

errc := make(chan error, 1)
go func() { errc <- srv.Serve(ln) }()
var runErr error
select {
case err := <-errc:
stopStrategy()
return err // the server stopped by itself: never a clean Shutdown
case <-sdone:
runErr = fmt.Errorf("strategy service stopped: %w", serr) // it only stops by itself on a failure
case <-ctx.Done():
}
shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := srv.Shutdown(shutdown); err != nil {
stopStrategy()
return fmt.Errorf("shutdown: %w", err)
}
if err := <-errc; err != nil && !errors.Is(err, http.ErrServerClosed) {
stopStrategy()
return err
}
return nil
stopStrategy()
return runErr
}

// errNoEngine is why a strategy run cannot start yet.
var errNoEngine = errors.New("strategy runs are not available: the engine is not attached to the strategy service")

// admitNoEngine refuses every registration until the engine is attached to the
// service (#703): the service is up and observable, but starts no run, so nothing
// can reach an account or a broker through it.
func admitNoEngine(context.Context, *proto.Register) (strategyserver.Grant, error) {
return strategyserver.Grant{}, strategyserver.Reject("strategy runs are not available yet: the engine is not attached to the strategy service (issue #703)")
}
95 changes: 92 additions & 3 deletions cmd/trader/serve_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,20 @@ import (
"go/parser"
"go/token"
"io"
"net"
"net/http"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"sync"
"testing"
"time"

"github.com/rustyeddy/trader/internal/store/storetest"
"github.com/rustyeddy/trader/strategy/proto"
strategyserver "github.com/rustyeddy/trader/strategy/server"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand All @@ -43,9 +47,21 @@ func (b *syncBuffer) String() string {
// stop function that interrupts it and returns what Execute returned.
func startServer(t *testing.T, env map[string]string, args ...string) (base string, stop func() error) {
t.Helper()
base, _, stop = startServerOut(t, env, args...)
return base, stop
}

// startServerOut is startServer that also returns what the command has printed so
// far. Unless the arguments name --strategy-listen it turns the strategy service
// off, so tests never fight over the default port.
func startServerOut(t *testing.T, env map[string]string, args ...string) (base string, out *syncBuffer, stop func() error) {
t.Helper()
if !slices.Contains(args, "--strategy-listen") {
args = append(args, "--strategy-listen", strategyOff)
}
a := newApp()
a.getenv = func(k string) string { return env[k] }
out := &syncBuffer{}
out = &syncBuffer{}
a.cmd.SetOut(out)
a.cmd.SetErr(io.Discard)
a.cmd.SetArgs(append([]string{"--log-output", filepath.Join(t.TempDir(), "serve.log"), "serve"}, args...))
Expand Down Expand Up @@ -75,7 +91,7 @@ func startServer(t *testing.T, env map[string]string, args ...string) (base stri
}
}
t.Cleanup(func() { _ = stop() })
return base, stop
return base, out, stop
}

func httpGet(t *testing.T, target string, header ...string) (int, string) {
Expand Down Expand Up @@ -156,6 +172,10 @@ func TestServe_RefusesWhatItShouldNotStartWith(t *testing.T) {
"a LAN address without a token": {nil, []string{"--listen", "192.168.1.5:0"}, envAPIToken},
"not an address": {nil, []string{"--listen", "nonsense"}, "--listen"},
"a zero timeout": {nil, []string{"--request-timeout", "0s"}, "--request-timeout"},
"strategy on every interface": {nil, []string{"--listen", "127.0.0.1:0", "--strategy-listen", "0.0.0.0:0"}, "--strategy-listen"},
"strategy on a LAN address": {nil, []string{"--listen", "127.0.0.1:0", "--strategy-listen", "192.168.1.5:0"}, "--strategy-listen"},
"strategy on an empty host": {nil, []string{"--listen", "127.0.0.1:0", "--strategy-listen", ":0"}, "--strategy-listen"},
"strategy not an address": {nil, []string{"--listen", "127.0.0.1:0", "--strategy-listen", "nonsense"}, "--strategy-listen"},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
Expand All @@ -182,7 +202,8 @@ func TestLoopback(t *testing.T) {

func TestServeCommand_IsTransportOnly(t *testing.T) {
const mod = "github.com/rustyeddy/trader"
allowed := map[string]bool{mod + "/service": true, mod + "/internal/store": true, mod + "/internal/logging": true, mod + "/rest": true}
allowed := map[string]bool{mod + "/service": true, mod + "/internal/store": true, mod + "/internal/logging": true, mod + "/rest": true,
mod + "/strategy/proto": true, mod + "/strategy/server": true}
f, err := parser.ParseFile(token.NewFileSet(), "serve.go", nil, parser.ImportsOnly)
require.NoError(t, err)
for _, imp := range f.Imports {
Expand All @@ -193,3 +214,71 @@ func TestServeCommand_IsTransportOnly(t *testing.T) {
}
}
}

func TestServe_TheStrategyServiceDefaultsToALoopbackPort(t *testing.T) {
f := newServeCmd(newApp()).Flags().Lookup("strategy-listen")

require.NotNil(t, f)
assert.Equal(t, defaultStrategyListen, f.DefValue)
local, err := loopback(defaultStrategyListen)
require.NoError(t, err)
assert.True(t, local, "the default is loopback")
assert.NotEqual(t, defaultListen, defaultStrategyListen, "it must not take the API's port")
}

func TestServe_TheStrategyServiceListensAndRefusesRegistrationsUntilTheEngineIsAttached(t *testing.T) {
root := storetest.Root(t)
_, out, stop := startServerOut(t, nil, "--config", writeConfig(t, ""), "--data-root", root, "--listen", "127.0.0.1:0", "--strategy-listen", "127.0.0.1:0")
re := regexp.MustCompile(`strategy protocol on tcp://(\S+)`)
require.Eventually(t, func() bool { return re.MatchString(out.String()) }, 5*time.Second, 10*time.Millisecond, "it announces its address")
addr := re.FindStringSubmatch(out.String())[1]

conn, err := net.Dial("tcp", addr)
require.NoError(t, err)
defer func() { _ = conn.Close() }()
reg := &proto.Register{Header: proto.Header{Seq: 1}, Name: "emax", Capabilities: []string{"bars"},
Run: proto.Run{Mode: proto.ModeBacktest, Interval: "H1", Instruments: []string{"EURUSD"},
From: time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2024, 4, 1, 0, 0, 0, 0, time.UTC)}}
require.NoError(t, conn.SetDeadline(time.Now().Add(5*time.Second)))
require.NoError(t, proto.Write(conn, reg, proto.DefaultMaxFrame))
m, err := proto.Read(conn, proto.DefaultMaxFrame)

require.NoError(t, err)
f, ok := m.(*proto.Failure)
require.True(t, ok, "expected an error message, got %T", m)
assert.Equal(t, proto.CodeRejected, f.Code)
assert.Contains(t, f.Message, "not available")
assert.Contains(t, f.Message, "#703")

require.NoError(t, stop(), "an interrupt stops both services cleanly")
_, derr := net.DialTimeout("tcp", addr, 500*time.Millisecond)
assert.Error(t, derr, "nothing listens on the strategy port any more")
}

func TestServe_TheStrategyServiceCanBeTurnedOff(t *testing.T) {
root := storetest.Root(t)
_, out, _ := startServerOut(t, nil, "--config", writeConfig(t, ""), "--data-root", root, "--listen", "127.0.0.1:0")

assert.Contains(t, out.String(), "strategy protocol off")
assert.NotContains(t, out.String(), "strategy protocol on")
}

func TestServe_AStrategyPortThatIsTakenIsAStartupError(t *testing.T) {
root := storetest.Root(t)
taken, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = taken.Close() }()

_, _, err = runData(t, nil, "serve", "--config", writeConfig(t, ""), "--data-root", root,
"--listen", "127.0.0.1:0", "--strategy-listen", taken.Addr().String())

require.Error(t, err)
assert.Contains(t, err.Error(), "--strategy-listen")
}

func TestAdmitNoEngine_RefusesEveryRegistrationWithAnExplanation(t *testing.T) {
_, err := admitNoEngine(context.Background(), &proto.Register{})

require.ErrorIs(t, err, strategyserver.ErrRejected)
assert.Contains(t, err.Error(), "not available")
}
9 changes: 9 additions & 0 deletions docs/DevelopersGuide.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,15 @@ build: raw partition --ReadSourcePartition--> marketdata.SourcePartition
uses `Load` with `SkipStale`). `rest` imports only `service` and the public
`marketdata` model, and no provider (a test enforces it); `cmd/trader/serve.go` reads
the environment (`TRADER_API_TOKEN`) and refuses a non-loopback address without it.
- `strategy`, `strategy/proto` and `strategy/server` are the public side of external
strategies (ADR-094). `strategy` is the seam a strategy implements and the requested
`Order`; `strategy/proto` is the wire contract and its strict codec; `strategy/server`
is the host: a loopback TCP listener, registration, the RunID and the lock-step
session. The server imports only `strategy` and `strategy/proto` (tests enforce it, and
that no guest package imports it); the engine is attached through the `Admit` and
`Handle` callbacks (#703). `trader serve` starts it on `127.0.0.1:8081`
(`--strategy-listen host:port` changes the address, `off` turns it off; a non-loopback
address is refused). Until the engine is attached every registration is refused.
- `service.MigrateLayout` (`trader data migrate`, ADR-092) moves `raw/<source>` to
`<source>/raw` with `store.Move` (one rename, never overwrites), the mapping in
`layout.LegacyRawDir` and verification by `store.Inventory`; it is a dry run unless
Expand Down
Loading
Loading