feat(rover): add drive profiles and leased remote command channel
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const boardLayoutSchema = "missioncore.board-layout/v1"
|
||||
|
||||
var boardSectionIDs = []string{"computer", "settings", "devices"}
|
||||
|
||||
type BoardLayout struct {
|
||||
Schema string `json:"schema"`
|
||||
Revision int64 `json:"revision"`
|
||||
OpenSections []string `json:"open_sections"`
|
||||
}
|
||||
|
||||
func validBoardSection(id string) bool {
|
||||
for _, section := range boardSectionIDs {
|
||||
if id == section {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *PresentationStore) readBoardLayout() (BoardLayout, error) {
|
||||
value := BoardLayout{Schema: boardLayoutSchema, OpenSections: append([]string{}, boardSectionIDs...)}
|
||||
file, err := os.Open(filepath.Join(p.dir, "board-layout.json"))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return value, nil
|
||||
}
|
||||
if err != nil {
|
||||
return BoardLayout{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
d := json.NewDecoder(io.LimitReader(file, 4097))
|
||||
d.DisallowUnknownFields()
|
||||
value = BoardLayout{}
|
||||
if d.Decode(&value) != nil || d.Decode(new(any)) != io.EOF || value.Schema != boardLayoutSchema || value.Revision < 0 || value.Revision >= 1<<53-1 || value.OpenSections == nil {
|
||||
return BoardLayout{}, errors.New("invalid board layout")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, id := range value.OpenSections {
|
||||
if !validBoardSection(id) || seen[id] {
|
||||
return BoardLayout{}, errors.New("invalid board section")
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (p *PresentationStore) saveBoardLayout(value BoardLayout) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.CreateTemp(p.dir, ".board-layout-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
if _, err = file.Write(append(data, '\n')); err == nil {
|
||||
err = file.Sync()
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if err = os.Rename(file.Name(), filepath.Join(p.dir, "board-layout.json")); err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(p.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
return dir.Sync()
|
||||
}
|
||||
|
||||
func (s *Server) boardLayoutRoutes(mux *http.ServeMux) {
|
||||
p := s.Presentation
|
||||
mux.HandleFunc("GET /api/presentation/board-layout", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(w, r) {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
value, err := p.readBoardLayout()
|
||||
if err != nil {
|
||||
reply(w, 500, map[string]string{"error": "Не удалось прочитать раскладку борта"})
|
||||
return
|
||||
}
|
||||
reply(w, 200, value)
|
||||
})
|
||||
mux.HandleFunc("PATCH /api/presentation/board-layout", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(w, r) {
|
||||
return
|
||||
}
|
||||
if r.Header.Get("Content-Type") != "application/json" {
|
||||
reply(w, 415, map[string]string{"error": "Ожидался JSON"})
|
||||
return
|
||||
}
|
||||
var change struct {
|
||||
Section string `json:"section"`
|
||||
Open *bool `json:"open"`
|
||||
}
|
||||
d := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024))
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&change) != nil || d.Decode(new(any)) != io.EOF || change.Open == nil || !validBoardSection(change.Section) {
|
||||
reply(w, 400, map[string]string{"error": "Некорректная раскладка борта"})
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
value, err := p.readBoardLayout()
|
||||
if err != nil {
|
||||
reply(w, 500, map[string]string{"error": "Не удалось прочитать раскладку борта"})
|
||||
return
|
||||
}
|
||||
next := []string{}
|
||||
for _, id := range boardSectionIDs {
|
||||
open := false
|
||||
for _, saved := range value.OpenSections {
|
||||
if saved == id {
|
||||
open = true
|
||||
}
|
||||
}
|
||||
if id == change.Section {
|
||||
open = *change.Open
|
||||
}
|
||||
if open {
|
||||
next = append(next, id)
|
||||
}
|
||||
}
|
||||
value.OpenSections = next
|
||||
value.Revision++
|
||||
if p.saveBoardLayout(value) != nil {
|
||||
reply(w, 500, map[string]string{"error": "Не удалось сохранить раскладку борта"})
|
||||
return
|
||||
}
|
||||
reply(w, 200, value)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBoardLayoutPersistsEmptyAndConcurrentSectionChanges(t *testing.T) {
|
||||
s, cookie := presentationServer(t)
|
||||
var wg sync.WaitGroup
|
||||
for _, id := range boardSectionIDs {
|
||||
wg.Add(1)
|
||||
go func(id string) {
|
||||
defer wg.Done()
|
||||
w := call(s, "PATCH", "/api/presentation/board-layout", `{"section":"`+id+`","open":false}`, cookie)
|
||||
if w.Code != 200 {
|
||||
t.Error(w.Code, w.Body.String())
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
wg.Wait()
|
||||
value, err := NewPresentationStore(s.Presentation.dir).readBoardLayout()
|
||||
if err != nil || value.Revision != 3 || len(value.OpenSections) != 0 || value.OpenSections == nil {
|
||||
t.Fatal(value, err)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(s.Presentation.dir, "board-layout.json"))
|
||||
if err != nil || info.Mode().Perm() != 0600 {
|
||||
t.Fatal(info, err)
|
||||
}
|
||||
if w := call(s, "GET", "/api/presentation/board-layout", "", nil); w.Code != 401 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoardLayoutRejectsInvalidAndPreservesCorruptFile(t *testing.T) {
|
||||
s, cookie := presentationServer(t)
|
||||
for _, body := range []string{`{"section":"motor","open":true}`, `{"section":"computer"}`, `{"section":"computer","open":"true"}`, `{"section":"computer","open":true,"extra":0}`} {
|
||||
if w := call(s, "PATCH", "/api/presentation/board-layout", body, cookie); w.Code != 400 {
|
||||
t.Fatal(w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
path := filepath.Join(s.Presentation.dir, "board-layout.json")
|
||||
raw := []byte(`{"schema":"missioncore.board-layout/v1","revision":0,"open_sections":["motor"]}`)
|
||||
if err := os.WriteFile(path, raw, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w := call(s, "PATCH", "/api/presentation/board-layout", `{"section":"computer","open":false}`, cookie); w.Code != 500 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
got, _ := os.ReadFile(path)
|
||||
if string(got) != string(raw) {
|
||||
t.Fatal("Corrupt file replaced")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package node
|
||||
|
||||
// Commands stream independently of telemetry. Only the local 20 Hz loop may
|
||||
// deliver the latest unexpired intent to the single VESC owner.
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (p *Pairing) roverBinding() *CoreBinding {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.state.Phase != "paired" || p.state.Binding == nil {
|
||||
return nil
|
||||
}
|
||||
b := *p.state.Binding
|
||||
return &b
|
||||
}
|
||||
func sameRoverBinding(a, b *CoreBinding) bool {
|
||||
return a != nil && b != nil && a.BindingID == b.BindingID && a.Endpoint == b.Endpoint && a.ClientPEM == b.ClientPEM && a.EndpointRevision == b.EndpointRevision
|
||||
}
|
||||
func roverWait(ctx context.Context, delay time.Duration) bool {
|
||||
if delay < 0 {
|
||||
delay = 0
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(delay):
|
||||
return true
|
||||
}
|
||||
}
|
||||
func (p *Pairing) roverChannel(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
binding := p.roverBinding()
|
||||
if binding != nil && p.Sensors != nil {
|
||||
p.runRoverChannel(ctx, *binding)
|
||||
}
|
||||
if !roverWait(ctx, time.Second) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
func (p *Pairing) runRoverChannel(parent context.Context, b CoreBinding) {
|
||||
config, err := bindingTLS(b, p.store.pairingKey())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
transport := func() *http.Transport {
|
||||
return &http.Transport{TLSClientConfig: config, Proxy: nil,
|
||||
MaxConnsPerHost: 1, MaxIdleConnsPerHost: 1, IdleConnTimeout: 10 * time.Second,
|
||||
TLSHandshakeTimeout: time.Second, ResponseHeaderTimeout: 2 * time.Second,
|
||||
DialContext: (&net.Dialer{Timeout: time.Second}).DialContext}
|
||||
}
|
||||
telemetry := &http.Client{Transport: transport(), Timeout: 300 * time.Millisecond, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect forbidden") }}
|
||||
stream := &http.Client{Transport: transport(), CheckRedirect: telemetry.CheckRedirect}
|
||||
defer telemetry.CloseIdleConnections()
|
||||
defer stream.CloseIdleConnections()
|
||||
random := make([]byte, 16)
|
||||
if _, err = rand.Read(random); err != nil {
|
||||
return
|
||||
}
|
||||
relay := hex.EncodeToString(random)
|
||||
id, _ := p.store.Public()
|
||||
base := map[string]any{"schema": PairSchema, "node_id": id, "binding_id": b.BindingID,
|
||||
"core_endpoint": b.Endpoint, "endpoint_revision": b.EndpointRevision, "relay_id": relay}
|
||||
state := newRoverStreamState(time.Now())
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
var workers sync.WaitGroup
|
||||
defer func() { cancel(); workers.Wait() }()
|
||||
workers.Add(2)
|
||||
go func() { defer workers.Done(); roverTelemetry(ctx, telemetry, b.Endpoint, base, state) }()
|
||||
go func() { defer workers.Done(); roverCommands(ctx, stream, b.Endpoint, base, state) }()
|
||||
var model *sensorModel
|
||||
for i := range sensorModels {
|
||||
if sensorModels[i].ID == "vesc.controller" {
|
||||
model = &sensorModels[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if model == nil {
|
||||
return
|
||||
}
|
||||
var diagnostic roverDiagnostics
|
||||
for ctx.Err() == nil && sameRoverBinding(&b, p.roverBinding()) {
|
||||
select {
|
||||
case <-state.wake:
|
||||
default:
|
||||
}
|
||||
started := time.Now()
|
||||
watch, command, stop := state.delivery(started)
|
||||
frame := roverFrame{}
|
||||
if command != nil {
|
||||
frame.Session, _ = command["id"].(string)
|
||||
frame.Sequence, _ = command["sequence"].(float64)
|
||||
frame.TTLMS, _ = command["ttl_ms"].(float64)
|
||||
}
|
||||
call, done := context.WithTimeout(ctx, 25*time.Millisecond)
|
||||
result, driverErr := p.Sensors.modelDriver(call, model, "/remote", map[string]any{"watch": watch, "command": command, "relay_id": relay})
|
||||
done()
|
||||
frame.DriverMS = time.Since(started).Milliseconds()
|
||||
if driverErr != nil {
|
||||
state.stopStream()
|
||||
frame.Stage = "driver_transport"
|
||||
state.delivered(map[string]any{}, 0)
|
||||
} else {
|
||||
state.delivered(result, stop)
|
||||
frame.State, _ = result["state"].(string)
|
||||
}
|
||||
diagnostic.record(started, frame)
|
||||
// New intent and stop signals bypass the periodic watchdog tick. The
|
||||
// buffered wake channel coalesces arrivals; it never queues commands.
|
||||
timer := time.NewTimer(max(0, 50*time.Millisecond-time.Since(started)))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-state.wake:
|
||||
case <-timer.C:
|
||||
}
|
||||
timer.Stop()
|
||||
}
|
||||
// Binding changes and channel shutdown explicitly retire any current intent.
|
||||
call, done := context.WithTimeout(context.Background(), 25*time.Millisecond)
|
||||
_, _ = p.Sensors.modelDriver(call, model, "/remote", map[string]any{"watch": false, "command": nil, "relay_id": relay})
|
||||
done()
|
||||
}
|
||||
func roverTelemetry(ctx context.Context, client *http.Client, endpoint string, base map[string]any, state *roverStreamState) {
|
||||
var diagnostic roverDiagnostics
|
||||
for ctx.Err() == nil {
|
||||
started := time.Now()
|
||||
snapshot, _ := state.view()
|
||||
payload := make(map[string]any, len(base)+1)
|
||||
for k, v := range base {
|
||||
payload[k] = v
|
||||
}
|
||||
payload["rover"] = snapshot
|
||||
raw, _ := json.Marshal(payload)
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", endpoint+"/v1/node/rover", bytes.NewReader(raw))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
frame := roverFrame{}
|
||||
frame.State, _ = snapshot["state"].(string)
|
||||
if frame.State == "preparing" || frame.State == "ready" || frame.State == "driving" || frame.State == "stopping" {
|
||||
frame.Session, _ = snapshot["session_id"].(string)
|
||||
}
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
frame.Stage = "telemetry_transport"
|
||||
} else {
|
||||
frame.Status = response.StatusCode
|
||||
var out roverStreamReply
|
||||
readErr := json.NewDecoder(io.LimitReader(response.Body, 32768)).Decode(&out)
|
||||
response.Body.Close()
|
||||
if readErr != nil || response.StatusCode != 200 {
|
||||
frame.Stage = "telemetry_response"
|
||||
} else if !state.sampleClock(out.Clock, started, time.Now()) {
|
||||
frame.Stage = "clock_invalid"
|
||||
state.stopStream()
|
||||
} else {
|
||||
state.setWatch(out.Watch)
|
||||
}
|
||||
}
|
||||
frame.CoreMS = time.Since(started).Milliseconds()
|
||||
diagnostic.record(started, frame)
|
||||
_, watch := state.view()
|
||||
delay := 500 * time.Millisecond
|
||||
if watch {
|
||||
delay = 100 * time.Millisecond
|
||||
}
|
||||
if !roverWait(ctx, delay-time.Since(started)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
func roverCommands(ctx context.Context, client *http.Client, endpoint string, base map[string]any, state *roverStreamState) {
|
||||
for ctx.Err() == nil {
|
||||
_, watch := state.view()
|
||||
if !watch {
|
||||
if !roverWait(ctx, 50*time.Millisecond) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
err := roverReadStream(ctx, client, endpoint, base, state)
|
||||
state.stopStream()
|
||||
if err != nil && ctx.Err() == nil {
|
||||
log.Printf("{\"event\":\"rover-command-stream\",\"state\":\"disconnected\"}")
|
||||
}
|
||||
if !roverWait(ctx, 250*time.Millisecond) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
func roverReadStream(parent context.Context, client *http.Client, endpoint string, base map[string]any, state *roverStreamState) error {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
defer cancel()
|
||||
raw, _ := json.Marshal(base)
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", endpoint+"/v1/node/rover-stream", bytes.NewReader(raw))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
sent := time.Now()
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != 200 {
|
||||
return errors.New("stream rejected")
|
||||
}
|
||||
// The native 200 ms output watchdog is unchanged. A silent stream also
|
||||
// cancels its network read and retires intent; reconnect never rearms it.
|
||||
guard := time.AfterFunc(350*time.Millisecond, cancel)
|
||||
defer guard.Stop()
|
||||
reader := bufio.NewReaderSize(response.Body, 32768)
|
||||
first := true
|
||||
for {
|
||||
line, err := reader.ReadSlice('\n')
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out roverStreamReply
|
||||
if json.Unmarshal(line, &out) != nil {
|
||||
return errors.New("invalid command frame")
|
||||
}
|
||||
if first {
|
||||
if !state.sampleClock(out.Clock, sent, time.Now()) {
|
||||
return errors.New("invalid stream clock")
|
||||
}
|
||||
first = false
|
||||
}
|
||||
if !state.accept(out) {
|
||||
return errors.New("invalid stream intent")
|
||||
}
|
||||
if !out.Watch {
|
||||
return nil
|
||||
}
|
||||
if !guard.Reset(350 * time.Millisecond) {
|
||||
return errors.New("stream deadline")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package node
|
||||
|
||||
// A bounded memory flight recorder. It never stores endpoints, certificates,
|
||||
// payloads or raw HTTP errors, and never changes admission or lease deadlines.
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
type roverFrame struct {
|
||||
AtMS int64 `json:"at_ms"`
|
||||
GapMS int64 `json:"gap_ms"`
|
||||
BindingMS int64 `json:"binding_ms"`
|
||||
CoreMS int64 `json:"core_ms"`
|
||||
DriverMS int64 `json:"driver_ms"`
|
||||
Status int `json:"http_status"`
|
||||
Stage string `json:"error_stage,omitempty"`
|
||||
Session string `json:"session,omitempty"`
|
||||
Sequence float64 `json:"sequence"`
|
||||
TTLMS float64 `json:"ttl_ms"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
type roverDiagnostics struct {
|
||||
frames []roverFrame
|
||||
last time.Time
|
||||
started time.Time
|
||||
state string
|
||||
session string
|
||||
emit func([]byte)
|
||||
}
|
||||
|
||||
func (d *roverDiagnostics) record(at time.Time, frame roverFrame) {
|
||||
if d.started.IsZero() {
|
||||
d.started = at
|
||||
}
|
||||
frame.AtMS = at.Sub(d.started).Milliseconds()
|
||||
if !d.last.IsZero() {
|
||||
frame.GapMS = at.Sub(d.last).Milliseconds()
|
||||
}
|
||||
d.last = at
|
||||
d.frames = append(d.frames, frame)
|
||||
if len(d.frames) > 128 {
|
||||
d.frames = d.frames[len(d.frames)-128:]
|
||||
}
|
||||
active := frame.Session != "" || d.session != ""
|
||||
changed := frame.State != d.state || frame.Session != d.session
|
||||
if active && (changed || frame.Stage != "" || frame.GapMS >= 200) {
|
||||
raw, _ := json.Marshal(struct {
|
||||
Event string `json:"event"`
|
||||
Frames []roverFrame `json:"frames"`
|
||||
}{"rover-channel", d.frames})
|
||||
if d.emit != nil {
|
||||
d.emit(raw)
|
||||
} else {
|
||||
log.Printf("%s", raw)
|
||||
}
|
||||
}
|
||||
d.state, d.session = frame.State, frame.Session
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRoverDiagnosticsRetainsCancellationCauseWithoutIdleLogSpam(t *testing.T) {
|
||||
var events [][]byte
|
||||
d := roverDiagnostics{emit: func(b []byte) { events = append(events, b) }}
|
||||
at := time.Unix(100, 0)
|
||||
for i := 0; i < 200; i++ {
|
||||
d.record(at.Add(time.Duration(i)*50*time.Millisecond), roverFrame{State: "observing"})
|
||||
}
|
||||
if len(events) != 0 || len(d.frames) != 128 {
|
||||
t.Fatal("idle must remain bounded and quiet")
|
||||
}
|
||||
at = at.Add(11 * time.Second)
|
||||
d.record(at, roverFrame{Session: "session", Sequence: 1, TTLMS: 350, State: "preparing"})
|
||||
d.record(at.Add(100*time.Millisecond), roverFrame{Session: "session", Sequence: 2, TTLMS: 320, State: "preparing"})
|
||||
if len(events) != 1 {
|
||||
t.Fatal("unchanged healthy frames should remain in memory")
|
||||
}
|
||||
d.record(at.Add(450*time.Millisecond), roverFrame{CoreMS: 300, Stage: "core_transport", State: "preparing"})
|
||||
if len(events) != 2 {
|
||||
t.Fatal("lost command needs diagnostic evidence")
|
||||
}
|
||||
var event struct {
|
||||
Frames []roverFrame `json:"frames"`
|
||||
}
|
||||
if err := json.Unmarshal(events[1], &event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
last := event.Frames[len(event.Frames)-1]
|
||||
if last.Stage != "core_transport" || last.GapMS != 350 || last.CoreMS != 300 || last.Session != "" {
|
||||
t.Fatal(last)
|
||||
}
|
||||
if event.Frames[len(event.Frames)-2].Sequence != 2 {
|
||||
t.Fatal("last accepted frame lost")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoverDiagnosticsRecordsBindingWaitAndDriverTerminalState(t *testing.T) {
|
||||
var events [][]byte
|
||||
d := roverDiagnostics{emit: func(b []byte) { events = append(events, b) }}
|
||||
at := time.Unix(100, 0)
|
||||
d.record(at, roverFrame{Session: "session", State: "preparing"})
|
||||
d.record(at.Add(500*time.Millisecond), roverFrame{Session: "session", State: "stopped", BindingMS: 400, DriverMS: 2})
|
||||
if len(events) != 2 {
|
||||
t.Fatal("terminal state must flush preceding transport history")
|
||||
}
|
||||
var event struct {
|
||||
Frames []roverFrame `json:"frames"`
|
||||
}
|
||||
if err := json.Unmarshal(events[1], &event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
last := event.Frames[len(event.Frames)-1]
|
||||
if last.BindingMS != 400 || last.DriverMS != 2 || last.State != "stopped" {
|
||||
t.Fatal(last)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type roverClock struct {
|
||||
Instance string `json:"instance"`
|
||||
MonotonicMS float64 `json:"monotonic_ms"`
|
||||
}
|
||||
type roverStreamReply struct {
|
||||
Watch bool `json:"watch"`
|
||||
Command map[string]any `json:"command"`
|
||||
Clock roverClock `json:"control_clock"`
|
||||
}
|
||||
|
||||
// Clock bounds use the request-send time, never RTT/2 or synchronized wall
|
||||
// clocks. Server time was sampled after that send, so it gives an upper bound
|
||||
// on server-minus-local monotonic offset even on an asymmetric network.
|
||||
type roverStreamState struct {
|
||||
mu sync.Mutex
|
||||
origin time.Time
|
||||
clockID string
|
||||
upperMS float64
|
||||
anchored time.Time
|
||||
clockSeen time.Time
|
||||
command map[string]any
|
||||
watch bool
|
||||
stopPending bool
|
||||
stopRevision uint64
|
||||
snapshot map[string]any
|
||||
retired map[string]bool
|
||||
inhibited bool
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
func newRoverStreamState(now time.Time) *roverStreamState {
|
||||
return &roverStreamState{origin: now, stopPending: true, stopRevision: 1, snapshot: map[string]any{}, retired: map[string]bool{}, wake: make(chan struct{}, 1)}
|
||||
}
|
||||
func finiteNumber(v any) (float64, bool) {
|
||||
n, ok := v.(float64)
|
||||
return n, ok && !math.IsNaN(n) && !math.IsInf(n, 0)
|
||||
}
|
||||
func (s *roverStreamState) sampleClock(c roverClock, sent, received time.Time) bool {
|
||||
if len(c.Instance) != 32 {
|
||||
return false
|
||||
}
|
||||
if _, err := hex.DecodeString(c.Instance); err != nil {
|
||||
return false
|
||||
}
|
||||
if math.IsNaN(c.MonotonicMS) || math.IsInf(c.MonotonicMS, 0) || c.MonotonicMS < 0 || received.Before(sent) {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
upper := c.MonotonicMS - float64(sent.Sub(s.origin).Microseconds())/1000 + 1
|
||||
if c.Instance != s.clockID {
|
||||
s.invalidate()
|
||||
s.clockID = c.Instance
|
||||
s.upperMS = upper
|
||||
s.anchored = received
|
||||
} else if s.clockSeen.IsZero() || received.Sub(s.clockSeen) > 2*time.Second || upper < s.upperMS+float64(received.Sub(s.anchored).Microseconds())/1e6 {
|
||||
s.upperMS = upper
|
||||
s.anchored = received
|
||||
}
|
||||
s.clockSeen = received
|
||||
return true
|
||||
}
|
||||
func (s *roverStreamState) accept(reply roverStreamReply) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if reply.Clock.Instance != s.clockID || s.clockID == "" {
|
||||
return false
|
||||
}
|
||||
if reply.Command == nil {
|
||||
if s.command != nil {
|
||||
s.invalidate()
|
||||
}
|
||||
return true
|
||||
}
|
||||
command := reply.Command
|
||||
if len(command) != 6 {
|
||||
return false
|
||||
}
|
||||
for _, k := range []string{"id", "sequence", "left", "right", "settings", "expires_mono_ms"} {
|
||||
if _, ok := command[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
expires, ok := finiteNumber(command["expires_mono_ms"])
|
||||
if !ok || expires > reply.Clock.MonotonicMS+400.001 {
|
||||
return false
|
||||
}
|
||||
id, ok := command["id"].(string)
|
||||
if !ok || len(id) != 32 {
|
||||
return false
|
||||
}
|
||||
if _, err := hex.DecodeString(id); err != nil {
|
||||
return false
|
||||
}
|
||||
sequence, ok := finiteNumber(command["sequence"])
|
||||
if !ok || sequence < 0 || sequence >= 1<<53 || math.Trunc(sequence) != sequence {
|
||||
return false
|
||||
}
|
||||
if s.inhibited || s.retired[id] {
|
||||
return true
|
||||
}
|
||||
// The native driver independently validates identity, settings and sequence.
|
||||
if s.stopPending {
|
||||
s.retire(id)
|
||||
return true
|
||||
}
|
||||
changed := s.command == nil || s.command["id"] != id || s.command["sequence"] != sequence
|
||||
s.command = command
|
||||
if changed {
|
||||
s.notify()
|
||||
}
|
||||
return true
|
||||
}
|
||||
func (s *roverStreamState) notify() {
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
func (s *roverStreamState) stopStream() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.invalidate()
|
||||
}
|
||||
func (s *roverStreamState) retire(id string) {
|
||||
if len(s.retired) >= 1024 {
|
||||
s.inhibited = true // Fail closed instead of forgetting interrupted sessions.
|
||||
return
|
||||
}
|
||||
s.retired[id] = true
|
||||
}
|
||||
func (s *roverStreamState) invalidate() {
|
||||
wake := !s.stopPending || s.command != nil
|
||||
if s.command != nil {
|
||||
if id, ok := s.command["id"].(string); ok {
|
||||
s.retire(id)
|
||||
}
|
||||
}
|
||||
s.command = nil
|
||||
s.stopPending = true
|
||||
s.stopRevision++
|
||||
if wake {
|
||||
s.notify()
|
||||
}
|
||||
}
|
||||
func (s *roverStreamState) delivery(now time.Time) (bool, map[string]any, uint64) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.stopPending {
|
||||
return s.watch, nil, s.stopRevision
|
||||
}
|
||||
if s.command == nil {
|
||||
return s.watch, nil, 0
|
||||
}
|
||||
if now.Sub(s.clockSeen) > 2*time.Second {
|
||||
s.invalidate()
|
||||
return s.watch, nil, s.stopRevision
|
||||
}
|
||||
expires, _ := finiteNumber(s.command["expires_mono_ms"])
|
||||
// Reserve 25 ms for the private driver RPC and 5 ms plus 1000 ppm for clock
|
||||
// quantization/rate uncertainty. Expiry is never extended by receipt time.
|
||||
serverUpper := float64(now.Sub(s.origin).Microseconds())/1000 + s.upperMS + 5 + float64(now.Sub(s.anchored).Microseconds())/1e6
|
||||
ttl := math.Min(400, expires-serverUpper-25)
|
||||
if ttl <= 0 {
|
||||
s.invalidate()
|
||||
return s.watch, nil, s.stopRevision
|
||||
}
|
||||
out := make(map[string]any, 6)
|
||||
for k, v := range s.command {
|
||||
if k != "expires_mono_ms" {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
out["ttl_ms"] = ttl
|
||||
return s.watch, out, 0
|
||||
}
|
||||
func (s *roverStreamState) delivered(snapshot map[string]any, stop uint64) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.snapshot = snapshot
|
||||
if stop != 0 && stop == s.stopRevision {
|
||||
s.stopPending = false
|
||||
}
|
||||
}
|
||||
func (s *roverStreamState) view() (map[string]any, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.snapshot, s.watch
|
||||
}
|
||||
func (s *roverStreamState) setWatch(watch bool) { s.mu.Lock(); s.watch = watch; s.mu.Unlock() }
|
||||
@@ -0,0 +1,235 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const testClockID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
func streamIntent(sequence int, serverNow, expires float64) roverStreamReply {
|
||||
return roverStreamReply{Watch: true, Clock: roverClock{testClockID, serverNow}, Command: map[string]any{
|
||||
"id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "sequence": float64(sequence), "left": float64(1), "right": float64(1),
|
||||
"settings": map[string]any{"standstill_confirmed": true, "current_a": float64(30), "max_erpm": float64(2000)}, "expires_mono_ms": expires}}
|
||||
}
|
||||
func acknowledgeStop(s *roverStreamState, at time.Time) {
|
||||
_, _, revision := s.delivery(at)
|
||||
s.delivered(map[string]any{"state": "observing"}, revision)
|
||||
}
|
||||
func TestRoverStreamAsymmetricDelayRetainsOriginalExpiry(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
// Core's clock is +10 seconds. Request took 70 ms outbound and 130 ms
|
||||
// inbound: dividing RTT in half would be incorrect on this connection.
|
||||
if !s.sampleClock(roverClock{testClockID, 10070}, origin, origin.Add(200*time.Millisecond)) {
|
||||
t.Fatal("clock")
|
||||
}
|
||||
acknowledgeStop(s, origin.Add(200*time.Millisecond))
|
||||
frame := streamIntent(1, 10200, 10600)
|
||||
if !s.accept(frame) {
|
||||
t.Fatal("intent")
|
||||
}
|
||||
_, command, _ := s.delivery(origin.Add(280 * time.Millisecond))
|
||||
if command == nil {
|
||||
t.Fatal("fresh streamed intent rejected")
|
||||
}
|
||||
ttl := command["ttl_ms"].(float64)
|
||||
if ttl <= 0 || 280+ttl > 600 {
|
||||
t.Fatal("delay extended original browser deadline", ttl)
|
||||
}
|
||||
// Replaying the same frame does not receive a new deadline.
|
||||
if !s.accept(frame) {
|
||||
t.Fatal("repeat")
|
||||
}
|
||||
_, late, _ := s.delivery(origin.Add(650 * time.Millisecond))
|
||||
if late != nil {
|
||||
t.Fatal("expired frame revived motor intent")
|
||||
}
|
||||
}
|
||||
func TestRoverStreamContinuousInputSurvivesMeasuredRelayJitter(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
s.sampleClock(roverClock{testClockID, 10050}, origin, origin.Add(150*time.Millisecond))
|
||||
acknowledgeStop(s, origin.Add(150*time.Millisecond))
|
||||
priorUntil := float64(0)
|
||||
for i, delay := range []int{60, 90, 45, 110, 80, 50, 100, 60, 90, 50} {
|
||||
sent := 200 + i*100
|
||||
arrival := sent + delay
|
||||
s.accept(streamIntent(i+1, float64(10000+sent), float64(10400+sent)))
|
||||
if i > 0 && float64(arrival) >= priorUntil {
|
||||
t.Fatal("lease gap under measured jitter", i, arrival, priorUntil)
|
||||
}
|
||||
_, command, _ := s.delivery(origin.Add(time.Duration(arrival) * time.Millisecond))
|
||||
if command == nil {
|
||||
t.Fatal("fresh command lost", i)
|
||||
}
|
||||
priorUntil = float64(arrival) + command["ttl_ms"].(float64)
|
||||
if priorUntil > float64(sent+400) {
|
||||
t.Fatal("end-to-end deadline expanded")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestRoverStreamDisconnectNeedsAcknowledgedStopBeforeNewFrames(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
s.sampleClock(roverClock{testClockID, 10000}, origin, origin)
|
||||
acknowledgeStop(s, origin)
|
||||
s.accept(streamIntent(1, 10000, 10400))
|
||||
s.stopStream()
|
||||
_, _, oldStop := s.delivery(origin)
|
||||
s.stopStream() // Another disconnect races the local driver's response.
|
||||
s.delivered(map[string]any{}, oldStop)
|
||||
s.accept(streamIntent(2, 10000, 10400))
|
||||
_, command, newStop := s.delivery(origin)
|
||||
if command != nil || newStop == 0 || newStop == oldStop {
|
||||
t.Fatal("stale ACK removed stop barrier")
|
||||
}
|
||||
s.delivered(map[string]any{}, newStop)
|
||||
_, command, _ = s.delivery(origin)
|
||||
if command != nil {
|
||||
t.Fatal("frame received before acknowledged stop was queued")
|
||||
}
|
||||
}
|
||||
func TestRoverStreamClockChangeAndStaleClockDisarm(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
s.sampleClock(roverClock{testClockID, 10000}, origin, origin)
|
||||
acknowledgeStop(s, origin)
|
||||
s.accept(streamIntent(1, 10000, 10400))
|
||||
s.sampleClock(roverClock{"cccccccccccccccccccccccccccccccc", 2}, origin, origin)
|
||||
_, command, stop := s.delivery(origin)
|
||||
if command != nil || stop == 0 {
|
||||
t.Fatal("Core restart retained intent")
|
||||
}
|
||||
if s.accept(streamIntent(2, 10000, 10400)) {
|
||||
t.Fatal("old clock accepted")
|
||||
}
|
||||
acknowledgeStop(s, origin)
|
||||
s.sampleClock(roverClock{testClockID, 10000}, origin, origin)
|
||||
acknowledgeStop(s, origin)
|
||||
// A fresh-looking frame alone cannot renew the clock calibration.
|
||||
fresh := streamIntent(3, 13000, 13400)
|
||||
fresh.Command["id"] = "dddddddddddddddddddddddddddddddd"
|
||||
s.accept(fresh)
|
||||
_, command, stop = s.delivery(origin.Add(3 * time.Second))
|
||||
if command != nil || stop == 0 {
|
||||
t.Fatal("stale clock allowed output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoverStreamUnsentIntentCannotStartAfterReconnect(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
s.sampleClock(roverClock{testClockID, 10000}, origin, origin)
|
||||
acknowledgeStop(s, origin)
|
||||
frame := streamIntent(1, 10000, 10400)
|
||||
s.accept(frame)
|
||||
// The network fails before the local driver ever sees this session.
|
||||
s.stopStream()
|
||||
acknowledgeStop(s, origin)
|
||||
s.accept(streamIntent(2, 10010, 10410))
|
||||
_, command, _ := s.delivery(origin)
|
||||
if command != nil {
|
||||
t.Fatal("undelivered old session started after reconnect")
|
||||
}
|
||||
frame.Command["id"] = "dddddddddddddddddddddddddddddddd"
|
||||
s.accept(frame)
|
||||
_, command, _ = s.delivery(origin)
|
||||
if command == nil {
|
||||
t.Fatal("new explicit session rejected after stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoverStreamIntentWakesImmediatelyAndCoalescesWithoutReplayWake(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
s.sampleClock(roverClock{testClockID, 10000}, origin, origin)
|
||||
acknowledgeStop(s, origin)
|
||||
for i := 1; i <= 3; i++ {
|
||||
s.accept(streamIntent(i, 10000, 10400))
|
||||
}
|
||||
if len(s.wake) != 1 {
|
||||
t.Fatal("intent notifications must coalesce")
|
||||
}
|
||||
<-s.wake
|
||||
_, command, _ := s.delivery(origin)
|
||||
if command["sequence"] != float64(3) {
|
||||
t.Fatal("queued an intermediate command")
|
||||
}
|
||||
s.accept(streamIntent(3, 10000, 10400))
|
||||
if len(s.wake) != 0 {
|
||||
t.Fatal("replayed frame woke driver again")
|
||||
}
|
||||
s.stopStream()
|
||||
if len(s.wake) != 1 {
|
||||
t.Fatal("stop did not wake driver")
|
||||
}
|
||||
<-s.wake
|
||||
s.stopStream()
|
||||
if len(s.wake) != 0 {
|
||||
t.Fatal("repeated failure would spin the local loop")
|
||||
}
|
||||
}
|
||||
func TestRoverStreamRejectsExtendedDeadlineAndUnknownCommandFields(t *testing.T) {
|
||||
origin := time.Unix(100, 0)
|
||||
s := newRoverStreamState(origin)
|
||||
s.sampleClock(roverClock{testClockID, 10000}, origin, origin)
|
||||
acknowledgeStop(s, origin)
|
||||
if s.accept(streamIntent(1, 10000, 10401)) {
|
||||
t.Fatal("extended deadline accepted")
|
||||
}
|
||||
frame := streamIntent(1, 10000, 10400)
|
||||
frame.Command["queued"] = true
|
||||
if s.accept(frame) {
|
||||
t.Fatal("unknown field accepted")
|
||||
}
|
||||
for _, sequence := range []any{map[string]any{}, []any{1}, "1", 1.5, float64(1 << 53)} {
|
||||
bad := streamIntent(1, 10000, 10400)
|
||||
bad.Command["sequence"] = sequence
|
||||
if s.accept(bad) {
|
||||
t.Fatal("malformed sequence accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestRoverStreamSilenceCancelsReadWithoutWaitingForTelemetry(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
json.NewEncoder(w).Encode(roverStreamReply{Watch: true, Clock: roverClock{testClockID, 10000}})
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
state := newRoverStreamState(time.Now())
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if roverReadStream(ctx, server.Client(), server.URL, map[string]any{}, state) == nil {
|
||||
t.Fatal("silent stream accepted")
|
||||
}
|
||||
if time.Since(started) > time.Second {
|
||||
t.Fatal("silent stream did not cancel its read")
|
||||
}
|
||||
}
|
||||
func TestRoverStreamReadsMultipleFramesOnOneResponse(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/node/rover-stream" {
|
||||
t.Error("wrong path")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
for i := 0; i < 3; i++ {
|
||||
raw, _ := json.Marshal(roverStreamReply{Watch: i < 2, Clock: roverClock{testClockID, 10000 + float64(i)}})
|
||||
fmt.Fprintln(w, string(raw))
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
state := newRoverStreamState(time.Now())
|
||||
if err := roverReadStream(context.Background(), server.Client(), server.URL, map[string]any{}, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user