- Add parser_map for market data refresh configuration propagation (cli/worker/api) - Implement refresh status model with SQLite persistence (status_store.go) - Add scheduler refresh status headless output to CLI operator - Extend backfill scheduler with status tracking (start/complete/fail) - Add socket events for scheduler refresh status (start/complete/fail) - Update proto definitions for refresh status - Add generated PB files for client
208 lines
6.8 KiB
Go
208 lines
6.8 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// RefreshStatusStore is a concurrent-safe in-memory store for scheduler runtime
|
|
// refresh status. The worker records a status after each tick; the socket
|
|
// handler reads all or a named schedule's status on demand.
|
|
type RefreshStatusStore struct {
|
|
mu sync.RWMutex
|
|
entries map[string]RefreshStatus
|
|
config Config
|
|
}
|
|
|
|
// NewRefreshStatusStore returns an empty, ready-to-use store.
|
|
func NewRefreshStatusStore() *RefreshStatusStore {
|
|
return &RefreshStatusStore{
|
|
entries: make(map[string]RefreshStatus),
|
|
}
|
|
}
|
|
|
|
// SetScheduleConfig stores the latest schedule config inside the store so the
|
|
// socket handler can compute next_run values without importing the scheduler
|
|
// package directly.
|
|
func (s *RefreshStatusStore) SetScheduleConfig(cfg Config) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.config = cfg
|
|
}
|
|
|
|
// ScheduleConfig returns the stored config (zero value if not set).
|
|
func (s *RefreshStatusStore) ScheduleConfig() Config {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
return s.config
|
|
}
|
|
|
|
// Record upserts the refresh status for the given schedule name.
|
|
func (s *RefreshStatusStore) Record(name string, status RefreshStatus) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.entries[name] = status
|
|
}
|
|
|
|
// All returns a snapshot of all recorded statuses, excluding internal entries.
|
|
func (s *RefreshStatusStore) All() []RefreshStatus {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
out := make([]RefreshStatus, 0, len(s.entries))
|
|
for k, v := range s.entries {
|
|
if k == "_config" {
|
|
continue
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Get returns the status for a named schedule. The second return value is false
|
|
// when no status has been recorded for that name yet.
|
|
func (s *RefreshStatusStore) Get(name string) (RefreshStatus, bool) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
v, ok := s.entries[name]
|
|
return v, ok
|
|
}
|
|
|
|
// RecordTick updates the store from a completed TickResult. Each item whose
|
|
// status is "succeeded" and has non-nil ImportSummary is aggregated into the
|
|
// status entry. Items with status "succeeded" but nil ImportSummary are treated
|
|
// as "ran but no evidence" — they count towards success classification but do
|
|
// NOT contribute bar counts, preventing fabricated S04 evidence.
|
|
//
|
|
// RecordTick only writes when at least one item belongs to the schedule.
|
|
// Zero-item runs are skipped so that no-item success, zero imported bar count,
|
|
// and next_run 누락 같은 false evidence is not created.
|
|
//
|
|
// Important: skipped/duplicate-window items are NOT treated as success. If any
|
|
// items exist but none succeeded (all skipped or all failed), the entry is
|
|
// marked "stale" (no error) or "error" (failed with message) rather than
|
|
// "success". A mixed result (some succeeded, some skipped/failed) is still
|
|
// "success" only when all succeeded AND no skipped/failed items exist; otherwise
|
|
// it is classified by the worst outcome. This prevents fabricating S04 evidence.
|
|
func (s *RefreshStatusStore) RecordTick(scheduleName string, items []ItemResult, now time.Time) {
|
|
if scheduleName == "" {
|
|
return
|
|
}
|
|
|
|
// Aggregate item outcomes for this schedule.
|
|
var lastErr string
|
|
totalImportedBars := 0
|
|
totalMissingBars := 0
|
|
totalGapBars := 0
|
|
totalDuplicateBars := 0
|
|
totalProviderDelayDays := 0
|
|
hasEvidence := false
|
|
succeededCount := 0
|
|
skippedCount := 0
|
|
ranNoEvidenceCount := 0 // succeeded but no ImportSummary
|
|
failedCount := 0
|
|
hasItems := false
|
|
|
|
for _, item := range items {
|
|
if item.Schedule != scheduleName {
|
|
continue
|
|
}
|
|
hasItems = true
|
|
switch item.Status {
|
|
case "succeeded":
|
|
succeededCount++
|
|
if item.ImportSummary != nil {
|
|
hasEvidence = true
|
|
totalImportedBars += item.ImportSummary.ImportedBars
|
|
totalMissingBars += item.ImportSummary.MissingBars
|
|
totalGapBars += item.ImportSummary.GapBars
|
|
totalDuplicateBars += item.ImportSummary.DuplicateBars
|
|
if item.ImportSummary.ProviderDelayDays > totalProviderDelayDays {
|
|
totalProviderDelayDays = item.ImportSummary.ProviderDelayDays
|
|
}
|
|
} else {
|
|
// Item succeeded but no import evidence — counts for
|
|
// classification but not for bar counts.
|
|
ranNoEvidenceCount++
|
|
}
|
|
case "skipped":
|
|
skippedCount++
|
|
case "failed":
|
|
failedCount++
|
|
if item.Error != "" {
|
|
lastErr = item.Error
|
|
}
|
|
}
|
|
}
|
|
|
|
// Skip recording when no items were found for this schedule.
|
|
// This prevents false evidence (no-item success, zero imported bar count, etc.).
|
|
if !hasItems {
|
|
return
|
|
}
|
|
|
|
// Determine status based on outcomes:
|
|
// - If any failed: status is "error"
|
|
// - If only skipped/duplicate (no succeeded, no failed): status is "stale"
|
|
// - If all succeeded but no evidence: status is "stale" (no-evidence success)
|
|
// - If all succeeded with at least one evidence: status is "success"
|
|
var statusStr string
|
|
if failedCount > 0 {
|
|
statusStr = "error"
|
|
} else if skippedCount > 0 || succeededCount == 0 {
|
|
// If there are skipped items or no succeeded items (but hasItems), don't claim success
|
|
statusStr = "stale"
|
|
} else if !hasEvidence {
|
|
// All items succeeded but no evidence — classify as "stale" rather than
|
|
// "success", preventing the silent "ran but unknown" → "success" escalation.
|
|
statusStr = "stale"
|
|
} else {
|
|
// All items succeeded AND at least one carried actual import evidence
|
|
statusStr = "success"
|
|
}
|
|
|
|
existing, hasExisting := s.Get(scheduleName)
|
|
|
|
var lastSuccess time.Time
|
|
// Only update last_success when we have actual evidence (ImportSummary).
|
|
// No-evidence success ("succeeded" but ImportSummary == nil) must NOT
|
|
// fabricate last_success evidence, preventing the "ran but unknown how
|
|
// much data" → "success" silent escalation.
|
|
if statusStr == "success" && hasEvidence {
|
|
lastSuccess = now
|
|
} else if hasExisting {
|
|
lastSuccess = existing.LastSuccess
|
|
}
|
|
|
|
// Accumulate counts only from actual evidence.
|
|
// For stale/error, preserve previous counts (don't add from partial/no-evidence runs).
|
|
var finalImportedBarCount, finalMissingCount, finalGapCount, finalDuplicateCount int
|
|
var finalProviderDelayDays int
|
|
if hasExisting {
|
|
finalImportedBarCount = existing.ImportedBarCount
|
|
finalMissingCount = existing.MissingCount
|
|
finalGapCount = existing.GapCount
|
|
finalDuplicateCount = existing.DuplicateCount
|
|
finalProviderDelayDays = existing.ProviderDelay
|
|
}
|
|
if statusStr == "success" && hasEvidence {
|
|
finalImportedBarCount += totalImportedBars
|
|
finalMissingCount += totalMissingBars
|
|
finalGapCount += totalGapBars
|
|
finalDuplicateCount += totalDuplicateBars
|
|
if totalProviderDelayDays > finalProviderDelayDays {
|
|
finalProviderDelayDays = totalProviderDelayDays
|
|
}
|
|
}
|
|
|
|
s.Record(scheduleName, RefreshStatus{
|
|
Schedule: scheduleName,
|
|
Status: statusStr,
|
|
LastSuccess: lastSuccess,
|
|
LastError: lastErr,
|
|
ImportedBarCount: finalImportedBarCount,
|
|
MissingCount: finalMissingCount,
|
|
GapCount: finalGapCount,
|
|
DuplicateCount: finalDuplicateCount,
|
|
ProviderDelay: finalProviderDelayDays,
|
|
})
|
|
}
|