- Add monthly bars aggregation in worker (services/worker/internal/marketdata/aggregation/) - Support monthly timeframe in operator runner and output - Add monthly aggregation test data and test cases - Update contracts (proto, Dart, Go) for monthly timeframe support - Sync parser_map, socket handlers, and worker client across services - Add code review and plan logs for multi-timeframe coverage milestones
541 lines
16 KiB
Go
541 lines
16 KiB
Go
package operator
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
)
|
|
|
|
// OutputFormat selects how runner events are rendered to stdout.
|
|
type OutputFormat string
|
|
|
|
const (
|
|
// OutputText emits grep-friendly key=value lines (the default).
|
|
OutputText OutputFormat = "text"
|
|
// OutputJSONL emits one JSON object per line.
|
|
OutputJSONL OutputFormat = "jsonl"
|
|
)
|
|
|
|
// noCount marks a step event that has no market result count (for example a
|
|
// hello handshake or a transport failure).
|
|
const noCount = -1
|
|
|
|
// StepEvent is one machine-readable record describing a single executed step.
|
|
type StepEvent struct {
|
|
Scenario string
|
|
Step string
|
|
Action string
|
|
Status string
|
|
Count int
|
|
ErrorCode string
|
|
ErrorMessage string
|
|
|
|
// Batch/matrix-specific output fields.
|
|
MatrixRunID string
|
|
StrategyID string
|
|
Timeframe string
|
|
PeriodID string
|
|
|
|
// Backtest-specific output fields.
|
|
RunID string
|
|
RunStatus string
|
|
StartingCash string
|
|
EndingEquity string
|
|
TotalReturn string
|
|
TradeCount int
|
|
|
|
// Import-specific output fields
|
|
Provider string
|
|
InstrumentCount int
|
|
BarCount int
|
|
|
|
// Monthly aggregation output fields. They render for aggregate_monthly_bars
|
|
// steps and prove SDD S04: deterministic monthly OHLCV with provenance. The
|
|
// counts default to noCount so an error step omits them; AggregationRuleID is
|
|
// the rule id reported in the response provenance.
|
|
SourceDailyBarCount int
|
|
MonthlyBarCount int
|
|
ProvenanceCount int
|
|
AggregationRuleID string
|
|
|
|
// Freshness-specific output fields
|
|
Universe string
|
|
FreshnessStatus string
|
|
LatestYYYYMMDD string
|
|
MissingCount int
|
|
MissingSymbols []string
|
|
|
|
// Gap-specific output fields
|
|
GapStatus string
|
|
GapCount int
|
|
GapYYYYMMDD []string
|
|
DuplicateCount int
|
|
DuplicateYYYYMMDD []string
|
|
ProviderDelayDays int
|
|
|
|
// Paper-trading-specific output fields. They are only rendered when
|
|
// AccountID is set, so non-paper steps never emit them.
|
|
AccountID string
|
|
Cash string
|
|
PositionCount int
|
|
FillCount int
|
|
Risk string
|
|
|
|
// Paper risk summary. Risk is "clear" when no order was rejected and
|
|
// "blocked" otherwise. RiskRejectedCount and RiskReason are only rendered for
|
|
// a blocked run so a clear run keeps stable, minimal output.
|
|
RiskRejectedCount int
|
|
RiskReason string
|
|
|
|
// Paper loop equity summary. EquityPointCount is the length of the paper
|
|
// equity curve and LatestEquity is the equity amount of its last point.
|
|
// They are omitted for an empty curve so loop smoke evidence stays stable.
|
|
EquityPointCount int
|
|
LatestEquity string
|
|
|
|
// Paper order lifecycle output fields. They render for submit/cancel/fill
|
|
// steps, keyed on OrderID, independently of the paper-state account block.
|
|
// OrderFillPrice/OrderFillCount carry the fill summary of a filled order.
|
|
OrderID string
|
|
OrderStatus string
|
|
OrderReason string
|
|
OrderFillPrice string
|
|
OrderFillCount int
|
|
|
|
// Live order lifecycle output fields. They render for submit/cancel/get live
|
|
// order steps, keyed on LiveOrderID. BrokerOrderID and BrokerStatus are the
|
|
// raw IDs and status strings the broker returned.
|
|
LiveOrderID string
|
|
LiveOrderStatus string
|
|
BrokerOrderID string
|
|
BrokerStatus string
|
|
OperatorConfirmed bool
|
|
|
|
// Live kill switch output fields. They render for get/set_live_kill_switch steps.
|
|
KillSwitchHalted bool
|
|
KillSwitchReason string
|
|
|
|
// Live risk policy output fields. They render for get_live_risk_policy steps.
|
|
MaxDailyOrders int
|
|
MaxOpenOrders int
|
|
AllowShortSelling bool
|
|
|
|
// Live risk decision output fields. They render for submit_live_order steps
|
|
// when a LiveRiskDecision is present in the response (i.e. risk-blocked submit).
|
|
HasRiskDecision bool
|
|
RiskDecisionAllow bool
|
|
RiskDecisionReason string
|
|
|
|
// Live account snapshot output fields. They render for sync_live_account and
|
|
// get_live_account_snapshot steps.
|
|
LiveAccountID string
|
|
LiveBroker string
|
|
CashCount int
|
|
LivePositionCount int
|
|
SyncedAtUnixMs int64
|
|
LiveStale bool
|
|
|
|
// Audit query output fields. They render for list_live_audit_events steps.
|
|
AuditEventCount int
|
|
}
|
|
|
|
// RunSummary is the final record describing a whole scenario run.
|
|
type RunSummary struct {
|
|
Scenario string
|
|
Status string
|
|
Steps int
|
|
Passed int
|
|
ExitCode int
|
|
}
|
|
|
|
// Writer renders runner events to an io.Writer in the selected format. stdout
|
|
// carries these machine-readable lines; scripts grep them for status keys.
|
|
type Writer struct {
|
|
w io.Writer
|
|
format OutputFormat
|
|
}
|
|
|
|
// NewWriter returns a Writer for the given format, defaulting to text.
|
|
func NewWriter(w io.Writer, format OutputFormat) *Writer {
|
|
if format == "" {
|
|
format = OutputText
|
|
}
|
|
return &Writer{w: w, format: format}
|
|
}
|
|
|
|
// WriteStep renders one executed step.
|
|
func (o *Writer) WriteStep(ev StepEvent) {
|
|
if o.format == OutputJSONL {
|
|
fields := map[string]any{
|
|
"type": "step",
|
|
"scenario": ev.Scenario,
|
|
"step": ev.Step,
|
|
"action": ev.Action,
|
|
"status": ev.Status,
|
|
}
|
|
if ev.Count >= 0 {
|
|
fields["count"] = ev.Count
|
|
}
|
|
if ev.ErrorCode != "" {
|
|
fields["error_code"] = ev.ErrorCode
|
|
}
|
|
if ev.ErrorMessage != "" {
|
|
fields["error_message"] = ev.ErrorMessage
|
|
}
|
|
if ev.MatrixRunID != "" {
|
|
fields["matrix_run_id"] = ev.MatrixRunID
|
|
fields["strategy_id"] = ev.StrategyID
|
|
fields["timeframe"] = ev.Timeframe
|
|
fields["period_id"] = ev.PeriodID
|
|
}
|
|
if ev.RunID != "" {
|
|
fields["run_id"] = ev.RunID
|
|
}
|
|
if ev.RunStatus != "" {
|
|
fields["run_status"] = ev.RunStatus
|
|
}
|
|
if ev.StartingCash != "" {
|
|
fields["starting_cash"] = ev.StartingCash
|
|
}
|
|
if ev.EndingEquity != "" {
|
|
fields["ending_equity"] = ev.EndingEquity
|
|
}
|
|
if ev.TotalReturn != "" {
|
|
fields["total_return"] = ev.TotalReturn
|
|
}
|
|
if ev.StartingCash != "" || ev.TradeCount > 0 {
|
|
fields["trade_count"] = ev.TradeCount
|
|
}
|
|
if ev.Provider != "" {
|
|
fields["provider"] = ev.Provider
|
|
}
|
|
if ev.InstrumentCount >= 0 {
|
|
fields["instrument_count"] = ev.InstrumentCount
|
|
}
|
|
if ev.BarCount >= 0 {
|
|
fields["bar_count"] = ev.BarCount
|
|
}
|
|
if ev.Action == string(ActionAggregateMonthlyBars) {
|
|
if ev.SourceDailyBarCount >= 0 {
|
|
fields["source_daily_bar_count"] = ev.SourceDailyBarCount
|
|
}
|
|
if ev.MonthlyBarCount >= 0 {
|
|
fields["monthly_bar_count"] = ev.MonthlyBarCount
|
|
}
|
|
if ev.ProvenanceCount >= 0 {
|
|
fields["provenance_count"] = ev.ProvenanceCount
|
|
}
|
|
if ev.AggregationRuleID != "" {
|
|
fields["aggregation_rule_id"] = ev.AggregationRuleID
|
|
}
|
|
}
|
|
if ev.Universe != "" {
|
|
fields["universe"] = ev.Universe
|
|
}
|
|
if ev.FreshnessStatus != "" {
|
|
fields["freshness_status"] = ev.FreshnessStatus
|
|
}
|
|
if ev.LatestYYYYMMDD != "" {
|
|
fields["latest_yyyymmdd"] = ev.LatestYYYYMMDD
|
|
}
|
|
if ev.MissingCount >= 0 {
|
|
fields["missing_count"] = ev.MissingCount
|
|
}
|
|
if len(ev.MissingSymbols) > 0 {
|
|
fields["missing_symbols"] = ev.MissingSymbols
|
|
}
|
|
if ev.GapStatus != "" {
|
|
fields["gap_status"] = ev.GapStatus
|
|
if ev.GapCount >= 0 {
|
|
fields["gap_count"] = ev.GapCount
|
|
}
|
|
if len(ev.GapYYYYMMDD) > 0 {
|
|
fields["gap_yyyymmdd"] = ev.GapYYYYMMDD
|
|
}
|
|
if ev.DuplicateCount >= 0 {
|
|
fields["duplicate_count"] = ev.DuplicateCount
|
|
}
|
|
if len(ev.DuplicateYYYYMMDD) > 0 {
|
|
fields["duplicate_yyyymmdd"] = ev.DuplicateYYYYMMDD
|
|
}
|
|
if ev.ProviderDelayDays >= 0 {
|
|
fields["provider_delay_days"] = ev.ProviderDelayDays
|
|
}
|
|
}
|
|
if ev.AccountID != "" {
|
|
fields["account_id"] = ev.AccountID
|
|
if ev.Cash != "" {
|
|
fields["cash"] = ev.Cash
|
|
}
|
|
fields["position_count"] = ev.PositionCount
|
|
fields["fill_count"] = ev.FillCount
|
|
if ev.Risk != "" {
|
|
fields["risk"] = ev.Risk
|
|
}
|
|
if ev.RiskRejectedCount > 0 {
|
|
fields["risk_rejected_count"] = ev.RiskRejectedCount
|
|
}
|
|
if ev.RiskReason != "" {
|
|
fields["risk_reason"] = ev.RiskReason
|
|
}
|
|
if ev.EquityPointCount > 0 {
|
|
fields["equity_point_count"] = ev.EquityPointCount
|
|
}
|
|
if ev.LatestEquity != "" {
|
|
fields["latest_equity"] = ev.LatestEquity
|
|
}
|
|
}
|
|
if ev.OrderID != "" {
|
|
fields["order_id"] = ev.OrderID
|
|
if ev.OrderStatus != "" {
|
|
fields["order_status"] = ev.OrderStatus
|
|
}
|
|
if ev.OrderReason != "" {
|
|
fields["order_reason"] = ev.OrderReason
|
|
}
|
|
if ev.Action == string(ActionFillPaperOrder) {
|
|
fields["fill_count"] = ev.OrderFillCount
|
|
}
|
|
if ev.OrderFillPrice != "" {
|
|
fields["fill_price"] = ev.OrderFillPrice
|
|
}
|
|
}
|
|
if ev.LiveOrderID != "" {
|
|
fields["live_order_id"] = ev.LiveOrderID
|
|
if ev.LiveOrderStatus != "" {
|
|
fields["live_order_status"] = ev.LiveOrderStatus
|
|
}
|
|
if ev.BrokerOrderID != "" {
|
|
fields["broker_order_id"] = ev.BrokerOrderID
|
|
}
|
|
if ev.BrokerStatus != "" {
|
|
fields["broker_status"] = ev.BrokerStatus
|
|
}
|
|
if ev.OperatorConfirmed {
|
|
fields["operator_confirmed"] = ev.OperatorConfirmed
|
|
}
|
|
}
|
|
if ev.Action == string(ActionGetLiveKillSwitch) || ev.Action == string(ActionSetLiveKillSwitch) {
|
|
fields["kill_switch_halted"] = ev.KillSwitchHalted
|
|
if ev.KillSwitchReason != "" {
|
|
fields["kill_switch_reason"] = ev.KillSwitchReason
|
|
}
|
|
}
|
|
if ev.Action == string(ActionGetLiveRiskPolicy) {
|
|
fields["max_daily_orders"] = ev.MaxDailyOrders
|
|
fields["max_open_orders"] = ev.MaxOpenOrders
|
|
fields["allow_short_selling"] = ev.AllowShortSelling
|
|
}
|
|
if ev.HasRiskDecision {
|
|
fields["risk_decision_allowed"] = ev.RiskDecisionAllow
|
|
if ev.RiskDecisionReason != "" {
|
|
fields["risk_decision_reason"] = ev.RiskDecisionReason
|
|
}
|
|
}
|
|
if ev.LiveAccountID != "" {
|
|
fields["live_account_id"] = ev.LiveAccountID
|
|
fields["live_broker"] = ev.LiveBroker
|
|
fields["cash_count"] = ev.CashCount
|
|
fields["live_position_count"] = ev.LivePositionCount
|
|
fields["synced_at_unix_ms"] = ev.SyncedAtUnixMs
|
|
fields["live_stale"] = ev.LiveStale
|
|
}
|
|
if ev.Action == string(ActionListLiveAuditEvents) {
|
|
fields["audit_event_count"] = ev.AuditEventCount
|
|
}
|
|
o.writeJSON(fields)
|
|
return
|
|
}
|
|
|
|
line := fmt.Sprintf("scenario=%s step=%s action=%s status=%s", ev.Scenario, ev.Step, ev.Action, ev.Status)
|
|
if ev.Count >= 0 {
|
|
line += fmt.Sprintf(" count=%d", ev.Count)
|
|
}
|
|
if ev.ErrorCode != "" {
|
|
line += fmt.Sprintf(" error.code=%s", ev.ErrorCode)
|
|
}
|
|
if ev.ErrorMessage != "" {
|
|
line += fmt.Sprintf(" error.message=%q", ev.ErrorMessage)
|
|
}
|
|
if ev.MatrixRunID != "" {
|
|
line += fmt.Sprintf(" matrix_run_id=%s strategy_id=%s timeframe=%s period_id=%s",
|
|
ev.MatrixRunID, ev.StrategyID, ev.Timeframe, ev.PeriodID)
|
|
}
|
|
if ev.RunID != "" {
|
|
line += fmt.Sprintf(" run.id=%s", ev.RunID)
|
|
}
|
|
if ev.RunStatus != "" {
|
|
line += fmt.Sprintf(" run.status=%s", ev.RunStatus)
|
|
}
|
|
if ev.StartingCash != "" {
|
|
line += fmt.Sprintf(" starting_cash=%s", ev.StartingCash)
|
|
}
|
|
if ev.EndingEquity != "" {
|
|
line += fmt.Sprintf(" ending_equity=%s", ev.EndingEquity)
|
|
}
|
|
if ev.TotalReturn != "" {
|
|
line += fmt.Sprintf(" total_return=%s", ev.TotalReturn)
|
|
}
|
|
if ev.StartingCash != "" || ev.TradeCount > 0 {
|
|
line += fmt.Sprintf(" trade_count=%d", ev.TradeCount)
|
|
}
|
|
if ev.Provider != "" {
|
|
line += fmt.Sprintf(" provider=%s", ev.Provider)
|
|
}
|
|
if ev.InstrumentCount >= 0 {
|
|
line += fmt.Sprintf(" instrument_count=%d", ev.InstrumentCount)
|
|
}
|
|
if ev.BarCount >= 0 {
|
|
line += fmt.Sprintf(" bar_count=%d", ev.BarCount)
|
|
}
|
|
if ev.Action == string(ActionAggregateMonthlyBars) {
|
|
if ev.SourceDailyBarCount >= 0 {
|
|
line += fmt.Sprintf(" source_daily_bar_count=%d", ev.SourceDailyBarCount)
|
|
}
|
|
if ev.MonthlyBarCount >= 0 {
|
|
line += fmt.Sprintf(" monthly_bar_count=%d", ev.MonthlyBarCount)
|
|
}
|
|
if ev.ProvenanceCount >= 0 {
|
|
line += fmt.Sprintf(" provenance_count=%d", ev.ProvenanceCount)
|
|
}
|
|
if ev.AggregationRuleID != "" {
|
|
line += fmt.Sprintf(" aggregation_rule_id=%s", ev.AggregationRuleID)
|
|
}
|
|
}
|
|
if ev.Universe != "" {
|
|
line += fmt.Sprintf(" universe=%s", ev.Universe)
|
|
}
|
|
if ev.FreshnessStatus != "" {
|
|
line += fmt.Sprintf(" freshness_status=%s", ev.FreshnessStatus)
|
|
}
|
|
if ev.LatestYYYYMMDD != "" {
|
|
line += fmt.Sprintf(" latest_yyyymmdd=%s", ev.LatestYYYYMMDD)
|
|
}
|
|
if ev.MissingCount >= 0 {
|
|
line += fmt.Sprintf(" missing_count=%d", ev.MissingCount)
|
|
}
|
|
if len(ev.MissingSymbols) > 0 {
|
|
line += fmt.Sprintf(" missing_symbols=%s", strings.Join(ev.MissingSymbols, ","))
|
|
}
|
|
if ev.GapStatus != "" {
|
|
line += fmt.Sprintf(" gap_status=%s", ev.GapStatus)
|
|
if ev.GapCount >= 0 {
|
|
line += fmt.Sprintf(" gap_count=%d", ev.GapCount)
|
|
}
|
|
if len(ev.GapYYYYMMDD) > 0 {
|
|
line += fmt.Sprintf(" gap_yyyymmdd=%s", strings.Join(ev.GapYYYYMMDD, ","))
|
|
}
|
|
if ev.DuplicateCount >= 0 {
|
|
line += fmt.Sprintf(" duplicate_count=%d", ev.DuplicateCount)
|
|
}
|
|
if len(ev.DuplicateYYYYMMDD) > 0 {
|
|
line += fmt.Sprintf(" duplicate_yyyymmdd=%s", strings.Join(ev.DuplicateYYYYMMDD, ","))
|
|
}
|
|
if ev.ProviderDelayDays >= 0 {
|
|
line += fmt.Sprintf(" provider_delay_days=%d", ev.ProviderDelayDays)
|
|
}
|
|
}
|
|
if ev.AccountID != "" {
|
|
line += fmt.Sprintf(" account_id=%s", ev.AccountID)
|
|
if ev.Cash != "" {
|
|
line += fmt.Sprintf(" cash=%s", ev.Cash)
|
|
}
|
|
line += fmt.Sprintf(" position_count=%d fill_count=%d", ev.PositionCount, ev.FillCount)
|
|
if ev.Risk != "" {
|
|
line += fmt.Sprintf(" risk=%s", ev.Risk)
|
|
}
|
|
if ev.RiskRejectedCount > 0 {
|
|
line += fmt.Sprintf(" risk_rejected_count=%d", ev.RiskRejectedCount)
|
|
}
|
|
if ev.RiskReason != "" {
|
|
line += fmt.Sprintf(" risk_reason=%q", ev.RiskReason)
|
|
}
|
|
if ev.EquityPointCount > 0 {
|
|
line += fmt.Sprintf(" equity_point_count=%d", ev.EquityPointCount)
|
|
}
|
|
if ev.LatestEquity != "" {
|
|
line += fmt.Sprintf(" latest_equity=%s", ev.LatestEquity)
|
|
}
|
|
}
|
|
if ev.OrderID != "" {
|
|
line += fmt.Sprintf(" order_id=%s", ev.OrderID)
|
|
if ev.OrderStatus != "" {
|
|
line += fmt.Sprintf(" order_status=%s", ev.OrderStatus)
|
|
}
|
|
if ev.OrderReason != "" {
|
|
line += fmt.Sprintf(" order_reason=%q", ev.OrderReason)
|
|
}
|
|
if ev.Action == string(ActionFillPaperOrder) {
|
|
line += fmt.Sprintf(" fill_count=%d", ev.OrderFillCount)
|
|
}
|
|
if ev.OrderFillPrice != "" {
|
|
line += fmt.Sprintf(" fill_price=%s", ev.OrderFillPrice)
|
|
}
|
|
}
|
|
if ev.LiveOrderID != "" {
|
|
line += fmt.Sprintf(" live_order_id=%s", ev.LiveOrderID)
|
|
if ev.LiveOrderStatus != "" {
|
|
line += fmt.Sprintf(" live_order_status=%s", ev.LiveOrderStatus)
|
|
}
|
|
if ev.BrokerOrderID != "" {
|
|
line += fmt.Sprintf(" broker_order_id=%s", ev.BrokerOrderID)
|
|
}
|
|
if ev.BrokerStatus != "" {
|
|
line += fmt.Sprintf(" broker_status=%s", ev.BrokerStatus)
|
|
}
|
|
if ev.OperatorConfirmed {
|
|
line += " operator_confirmed=true"
|
|
}
|
|
}
|
|
if ev.Action == string(ActionGetLiveKillSwitch) || ev.Action == string(ActionSetLiveKillSwitch) {
|
|
line += fmt.Sprintf(" kill_switch_halted=%v", ev.KillSwitchHalted)
|
|
if ev.KillSwitchReason != "" {
|
|
line += fmt.Sprintf(" kill_switch_reason=%q", ev.KillSwitchReason)
|
|
}
|
|
}
|
|
if ev.Action == string(ActionGetLiveRiskPolicy) {
|
|
line += fmt.Sprintf(" max_daily_orders=%d max_open_orders=%d allow_short_selling=%v", ev.MaxDailyOrders, ev.MaxOpenOrders, ev.AllowShortSelling)
|
|
}
|
|
if ev.HasRiskDecision {
|
|
line += fmt.Sprintf(" risk_decision_allowed=%v", ev.RiskDecisionAllow)
|
|
if ev.RiskDecisionReason != "" {
|
|
line += fmt.Sprintf(" risk_decision_reason=%q", ev.RiskDecisionReason)
|
|
}
|
|
}
|
|
if ev.LiveAccountID != "" {
|
|
line += fmt.Sprintf(" live_account_id=%s live_broker=%s cash_count=%d live_position_count=%d synced_at_unix_ms=%d live_stale=%v",
|
|
ev.LiveAccountID, ev.LiveBroker, ev.CashCount, ev.LivePositionCount, ev.SyncedAtUnixMs, ev.LiveStale)
|
|
}
|
|
if ev.Action == string(ActionListLiveAuditEvents) {
|
|
line += fmt.Sprintf(" audit_event_count=%d", ev.AuditEventCount)
|
|
}
|
|
fmt.Fprintln(o.w, line)
|
|
}
|
|
|
|
// WriteSummary renders the final run summary.
|
|
func (o *Writer) WriteSummary(s RunSummary) {
|
|
if o.format == OutputJSONL {
|
|
o.writeJSON(map[string]any{
|
|
"type": "summary",
|
|
"scenario": s.Scenario,
|
|
"status": s.Status,
|
|
"steps": s.Steps,
|
|
"passed": s.Passed,
|
|
"exit_code": s.ExitCode,
|
|
})
|
|
return
|
|
}
|
|
fmt.Fprintf(o.w, "scenario=%s status=%s steps=%d passed=%d exit_code=%d\n", s.Scenario, s.Status, s.Steps, s.Passed, s.ExitCode)
|
|
}
|
|
|
|
func (o *Writer) writeJSON(fields map[string]any) {
|
|
data, err := json.Marshal(fields)
|
|
if err != nil {
|
|
// Marshalling a map of strings/ints cannot realistically fail; fall back
|
|
// to a minimal line so a step result is never silently dropped.
|
|
fmt.Fprintf(o.w, "{\"type\":\"error\",\"error_message\":%q}\n", err.Error())
|
|
return
|
|
}
|
|
fmt.Fprintln(o.w, string(data))
|
|
}
|