{
  "format": "siebly-agent-recipe/v1",
  "id": "historical-live-data-pipeline-core",
  "title": "Historical Backfill with Live WebSocket Streams",
  "lastReviewed": "2026-05-12",
  "package": {
    "ecosystem": "npm",
    "name": "selected exchange SDK",
    "install": "Install the maintained SDK for the selected exchange",
    "docs": "https://siebly.io/sdk",
    "repository": "Use the selected SDK repository and installed package source"
  },
  "scope": {
    "runtime": "Node.js LTS",
    "language": "TypeScript",
    "dataFamilies": [
      "candles and klines",
      "trades and aggregate trades",
      "order book snapshots plus deltas",
      "funding, mark, index, open-interest, and other periodically sampled market data",
      "private account data when the exchange exposes a trusted historical read plus live updates"
    ],
    "defaultMode": "public or read-only data pipeline with no order placement",
    "credentials": "public endpoints only unless the selected data family requires read-only account keys"
  },
  "requiredSources": [
    "https://siebly.io/ai/historical-live-data-pipeline",
    "https://siebly.io/.well-known/recipes/historical-live-data-pipeline-core.json",
    "https://siebly.io/.well-known/conformance/historical-live-data/latest.json",
    "https://siebly.io/.well-known/conformance/historical-live-data/v1/schema.json",
    "https://siebly.io/.well-known/conformance/historical-live-data/v1/fixtures.json",
    "https://siebly.io/.well-known/conformance/historical-live-data/v1/runner.ts",
    "https://siebly.io/ai/patterns",
    "https://siebly.io/ai",
    "https://siebly.io/sdk",
    "https://siebly.io/.well-known/agent-skills/siebly-crypto-exchange-api/SKILL.md",
    "https://siebly.io/reference/glossary",
    "https://siebly.io/llms.txt",
    "https://siebly.io/llms-tasks.txt",
    "https://siebly.io/llms-full.txt",
    "https://siebly.io/.well-known/siebly-sdk-catalog.json"
  ],
  "workflow": [
    "Define the data identity up front: product scope, symbol or instrument, stream name, historical REST endpoint, primary timestamp, optional sequence/update ID, and replay key.",
    "Open the WebSocket and send the subscription request before historical backfill when the exchange supports it.",
    "Wait for the package-specific [subscription acknowledgement](https://siebly.io/reference/glossary#subscription-acknowledgement) before treating live events as authoritative workflow input.",
    "Buffer raw live events with local receive time while REST history or account-state hydration is running.",
    "Backfill historical records into a local store keyed by the normalized product, symbol, interval or stream identity, and event timestamp or update ID.",
    "Replay buffered events deterministically, skip stale or duplicate records, and keep one normalized state transition path for REST rows and WebSocket events.",
    "Enable downstream live workflows only after [subscription acknowledgement](https://siebly.io/reference/glossary#subscription-acknowledgement), backfill, buffered replay, and readiness state are complete.",
    "If the data type has a finality signal, run strategy, indicator, signal generation, optional external alert, [order-intent](https://siebly.io/reference/glossary#order-intent), or account-decision workflows only after that final/closed/terminal signal.",
    "After reconnect, pause correctness-sensitive workflows, keep buffering where possible, resubscribe or confirm subscriptions, run scoped REST resync, replay buffered events, and then re-enable live processing."
  ],
  "lifecycleChain": [
    "config scope",
    "product/symbol/stream identity",
    "SDK REST/read method",
    "SDK WebSocket subscription",
    "subscription acknowledgement",
    "buffered live event",
    "historical REST/read row",
    "normalized store key",
    "dedupe or replay decision",
    "readiness state",
    "downstream side effect",
    "reconnect/resync boundary"
  ],
  "validationFixtures": [
    "startup_requires_ack_backfill_replay_before_live_processing",
    "subscribe_return_or_socket_open_does_not_enable_workflow",
    "buffered_live_event_replays_after_backfill_once",
    "duplicate_historical_and_live_record_is_deduped",
    "out_of_order_live_events_are_replayed_deterministically",
    "finality_required_before_downstream_side_effect",
    "reconnect_pauses_workflow_until_scoped_resync_completes",
    "sample_symbol_interval_or_category_is_not_a_hidden_runtime_default",
    "orderbook_sequence_gap_blocks_downstream_until_snapshot_resync",
    "private_read_plus_stream_requires_read_only_credentials_and_hydration_gate"
  ],
  "completionGate": "Do not mark the pipeline complete until three consecutive full data-lifecycle review passes produce no code, tests, fixtures, or documentation changes.",
  "dataFamilyOverlays": [
    {
      "id": "candle-kline-ohlcv-overlay",
      "title": "Candle/Kline/OHLCV Overlay",
      "appliesTo": [
        "candles",
        "klines",
        "OHLCV bars"
      ],
      "exchangeFactsToVerify": [
        "selected product family or category name",
        "REST candle/history method, request fields, pagination, and timestamp order",
        "public WebSocket candle/kline topic or subscription helper",
        "subscription acknowledgement event and payload shape",
        "final or closed candle field and closed value",
        "symbol, interval, start time, close time, OHLCV, volume, and trade-count fields where available",
        "reconnect or resubscribe hooks",
        "shutdown method for the SDK WebSocket client"
      ],
      "rules": [
        "Treat examples as request-shape references only; do not copy sample symbols, intervals, categories, or product families as runtime defaults.",
        "Keep symbol, interval, product family, and category configurable.",
        "Normalize REST rows and WebSocket updates into one candle shape keyed by product family, symbol, interval, and candle start time.",
        "Open candles may update local state, but cannot trigger strategy, indicator, signal generation, optional external alert, [order-intent](https://siebly.io/reference/glossary#order-intent), or account-decision workflows.",
        "Run downstream logic only when the exchange or SDK marks the candle final or closed.",
        "Deduplicate final candles so a reconnect, replay, or repeated final update cannot run the same workflow twice."
      ]
    }
  ],
  "stateGates": [
    "transport_open is useful telemetry but not workflow readiness",
    "subscription_request_sent records intent but does not prove [subscription acknowledgement](https://siebly.io/reference/glossary#subscription-acknowledgement)",
    "subscription_acknowledged is required before buffered live events can become authoritative",
    "rest_backfill_complete or scoped_hydration_complete is required before live workflow enablement",
    "buffered_replay_complete is required before downstream side effects",
    "live_processing_enabled is a local readiness flag after reconciliation"
  ],
  "reviewRules": [
    "Trace the full data lifecycle from config scope to downstream side effect before claiming readiness.",
    "Every correctness claim in README, prompt, or docs needs a fixture or replay case. Behaviors without fixtures should be documented as unsupported or unverified.",
    "Do not mark the pipeline complete until three consecutive full data-lifecycle review passes produce no code, tests, fixtures, or documentation changes."
  ],
  "implementationPages": [
    "https://siebly.io/ai/candle-pipeline/binance",
    "https://siebly.io/ai/candle-pipeline/bybit"
  ],
  "safety": [
    "This core pattern avoids exchange-specific topic names, payload fields, and SDK method names.",
    "Exchange implementation pages should reference this lifecycle and add only package-specific method names, acknowledgement shapes, finality fields, reconnect hooks, and shutdown APIs.",
    "Public-only pipelines must not add private clients, account reads, order placement, cancellation, or amendment.",
    "Private data pipelines must use scoped read-only keys unless the user explicitly requests an order-capable workflow with EXECUTION_MODE=DRY_RUN_PRIVATE, DEMO, TESTNET, or LIVE."
  ],
  "promptFrameworkCompletionWorkflow": {
    "instruction": "Recursive completion workflow:\n1. Before implementation, save this exact prompt in docs/AI_PROMPT.md (or docs/SPEC.md when that is the project standard) and write docs/PLAN.md with phases, invariants, tests or fixtures, docs to update, and acceptance gates.\n2. Review docs/PLAN.md for missing workflows, unsafe assumptions, product/exchange-specific leakage, unclear state ownership, confirmation or recovery gaps, missing tests, and incomplete docs. Update docs/PLAN.md and repeat until one full review pass finds no actionable changes.\n3. Implement one plan phase at a time. After each phase, review changed code, tests, fixtures, docs, generated artifacts, and runtime workflows against docs/PLAN.md and this prompt. Fix gaps and repeat until that phase has no actionable changes before starting the next phase.\n4. After all phases, run a full-depth project review across every workflow, lifecycle, state transition, error path, and artifact. This is not a shallow summary pass. Fix every actionable gap and repeat until a full pass finds no further changes, then record the final review outcome in docs/PLAN.md.",
    "acceptanceRequirement": "docs/PLAN.md records the initial plan, plan-review iterations, phase review outcomes, final full-project review, validation commands, and any documented non-claims. No plan phase or project completion is accepted until the recursive review loop finds no actionable gaps, flaws, or incomplete workflows left to correct."
  }
}