rara/db/migrations/00001_initial.sql

-- +goose Up

CREATE TABLE projects (
    id uuid PRIMARY KEY,
    slug text NOT NULL UNIQUE,
    name text NOT NULL,
    description text NOT NULL DEFAULT '',
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now()
);

CREATE TABLE integration_instances (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    name text NOT NULL,
    kind text NOT NULL CHECK (kind IN (
        'source',
        'retrieval_backend',
        'artifact_store',
        'embedding_provider',
        'rerank_provider',
        'generation_provider',
        'executor'
    )),
    implementation text NOT NULL,
    schema_version text NOT NULL DEFAULT 'v1',
    capabilities jsonb NOT NULL DEFAULT '{}'::jsonb,
    config jsonb NOT NULL DEFAULT '{}'::jsonb,
    secret_ref text,
    enabled boolean NOT NULL DEFAULT true,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, name)
);

CREATE TABLE knowledge_sources (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    integration_instance_id uuid NOT NULL REFERENCES integration_instances(id),
    key text NOT NULL,
    name text NOT NULL,
    revision text,
    cursor jsonb NOT NULL DEFAULT '{}'::jsonb,
    sync_state text NOT NULL DEFAULT 'pending'
        CHECK (sync_state IN ('pending', 'syncing', 'ready', 'failed', 'disabled')),
    last_synced_at timestamptz,
    tombstone_checked_at timestamptz,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, key)
);

CREATE TABLE knowledge_bases (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    key text NOT NULL,
    name text NOT NULL,
    description text NOT NULL DEFAULT '',
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, key)
);

CREATE TABLE knowledge_base_sources (
    knowledge_base_id uuid NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
    knowledge_source_id uuid NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
    created_at timestamptz NOT NULL DEFAULT now(),
    PRIMARY KEY (knowledge_base_id, knowledge_source_id)
);

CREATE TABLE artifacts (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    kind text NOT NULL,
    uri text NOT NULL,
    content_hash text NOT NULL,
    size_bytes bigint NOT NULL CHECK (size_bytes >= 0),
    media_type text NOT NULL DEFAULT 'application/octet-stream',
    schema_version text NOT NULL DEFAULT 'v1',
    metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
    status text NOT NULL DEFAULT 'ready'
        CHECK (status IN ('pending', 'ready', 'failed', 'deleted')),
    created_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, content_hash, kind)
);

CREATE TABLE artifact_lineage (
    parent_artifact_id uuid NOT NULL REFERENCES artifacts(id) ON DELETE CASCADE,
    child_artifact_id uuid NOT NULL REFERENCES artifacts(id) ON DELETE CASCADE,
    relation text NOT NULL,
    created_at timestamptz NOT NULL DEFAULT now(),
    PRIMARY KEY (parent_artifact_id, child_artifact_id, relation),
    CHECK (parent_artifact_id <> child_artifact_id)
);

CREATE TABLE datasets (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    key text NOT NULL,
    name text NOT NULL,
    purpose text NOT NULL CHECK (purpose IN (
        'raw_snapshot',
        'normalized',
        'retrieval',
        'evaluation',
        'training_candidate',
        'training',
        'validation',
        'holdout'
    )),
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, key)
);

CREATE TABLE workflow_definitions (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    key text NOT NULL,
    name text NOT NULL,
    kind text NOT NULL,
    created_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, key)
);

CREATE TABLE workflow_versions (
    id uuid PRIMARY KEY,
    workflow_definition_id uuid NOT NULL REFERENCES workflow_definitions(id) ON DELETE CASCADE,
    version integer NOT NULL CHECK (version > 0),
    spec jsonb NOT NULL,
    created_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (workflow_definition_id, version)
);

CREATE TABLE schedules (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    workflow_version_id uuid NOT NULL REFERENCES workflow_versions(id),
    name text NOT NULL,
    cron_expression text NOT NULL,
    timezone text NOT NULL DEFAULT 'UTC',
    enabled boolean NOT NULL DEFAULT true,
    input jsonb NOT NULL DEFAULT '{}'::jsonb,
    next_run_at timestamptz,
    last_run_at timestamptz,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, name)
);

CREATE TABLE workflow_runs (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    workflow_version_id uuid NOT NULL REFERENCES workflow_versions(id),
    status text NOT NULL DEFAULT 'pending'
        CHECK (status IN ('pending', 'running', 'succeeded', 'failed', 'cancelled')),
    trigger_type text NOT NULL,
    input jsonb NOT NULL DEFAULT '{}'::jsonb,
    output jsonb NOT NULL DEFAULT '{}'::jsonb,
    error jsonb NOT NULL DEFAULT '{}'::jsonb,
    idempotency_key text,
    started_at timestamptz,
    finished_at timestamptz,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now()
);

CREATE UNIQUE INDEX workflow_runs_idempotency_idx
    ON workflow_runs (project_id, idempotency_key)
    WHERE idempotency_key IS NOT NULL;

CREATE TABLE step_runs (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    workflow_run_id uuid NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE,
    step_key text NOT NULL,
    executor_kind text NOT NULL,
    status text NOT NULL DEFAULT 'pending'
        CHECK (status IN (
            'pending',
            'leased',
            'running',
            'succeeded',
            'retryable_failed',
            'terminal_failed',
            'cancelled'
        )),
    attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
    max_attempts integer NOT NULL DEFAULT 3 CHECK (max_attempts > 0),
    priority integer NOT NULL DEFAULT 0,
    input jsonb NOT NULL DEFAULT '{}'::jsonb,
    output jsonb NOT NULL DEFAULT '{}'::jsonb,
    error jsonb NOT NULL DEFAULT '{}'::jsonb,
    idempotency_key text NOT NULL,
    available_at timestamptz NOT NULL DEFAULT now(),
    leased_by text,
    leased_until timestamptz,
    started_at timestamptz,
    finished_at timestamptz,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, idempotency_key)
);

CREATE INDEX step_runs_lease_idx
    ON step_runs (executor_kind, status, available_at, priority DESC, created_at)
    WHERE status IN ('pending', 'retryable_failed');

CREATE INDEX step_runs_workflow_idx ON step_runs (workflow_run_id, created_at);

CREATE TABLE dataset_versions (
    id uuid PRIMARY KEY,
    dataset_id uuid NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
    version integer NOT NULL CHECK (version > 0),
    artifact_id uuid NOT NULL REFERENCES artifacts(id),
    source_revision text,
    recipe jsonb NOT NULL DEFAULT '{}'::jsonb,
    stats jsonb NOT NULL DEFAULT '{}'::jsonb,
    status text NOT NULL DEFAULT 'candidate'
        CHECK (status IN ('candidate', 'active', 'retired', 'failed')),
    created_by_step_run_id uuid REFERENCES step_runs(id),
    created_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (dataset_id, version)
);

CREATE TABLE evaluation_suites (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    key text NOT NULL,
    name text NOT NULL,
    description text NOT NULL DEFAULT '',
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (project_id, key)
);

CREATE TABLE evaluation_cases (
    id uuid PRIMARY KEY,
    evaluation_suite_id uuid NOT NULL REFERENCES evaluation_suites(id) ON DELETE CASCADE,
    case_key text NOT NULL,
    input jsonb NOT NULL,
    expectations jsonb NOT NULL DEFAULT '{}'::jsonb,
    tags text[] NOT NULL DEFAULT ARRAY[]::text[],
    enabled boolean NOT NULL DEFAULT true,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (evaluation_suite_id, case_key)
);

CREATE TABLE evaluation_runs (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    evaluation_suite_id uuid NOT NULL REFERENCES evaluation_suites(id),
    subject_type text NOT NULL,
    subject_id uuid NOT NULL,
    baseline_subject_id uuid,
    workflow_run_id uuid REFERENCES workflow_runs(id),
    status text NOT NULL DEFAULT 'pending'
        CHECK (status IN ('pending', 'running', 'succeeded', 'failed', 'cancelled')),
    metrics jsonb NOT NULL DEFAULT '{}'::jsonb,
    passed boolean,
    started_at timestamptz,
    finished_at timestamptz,
    created_at timestamptz NOT NULL DEFAULT now()
);

CREATE TABLE evaluation_results (
    id uuid PRIMARY KEY,
    evaluation_run_id uuid NOT NULL REFERENCES evaluation_runs(id) ON DELETE CASCADE,
    evaluation_case_id uuid REFERENCES evaluation_cases(id) ON DELETE CASCADE,
    metric_key text NOT NULL,
    score double precision,
    passed boolean,
    details jsonb NOT NULL DEFAULT '{}'::jsonb,
    created_at timestamptz NOT NULL DEFAULT now()
);

CREATE INDEX evaluation_results_run_idx ON evaluation_results (evaluation_run_id);

CREATE TABLE releases (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    knowledge_base_id uuid NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
    alias text NOT NULL DEFAULT 'production',
    version integer NOT NULL CHECK (version > 0),
    status text NOT NULL DEFAULT 'candidate'
        CHECK (status IN ('candidate', 'active', 'retired', 'failed')),
    dataset_version_id uuid NOT NULL REFERENCES dataset_versions(id),
    index_artifact_id uuid REFERENCES artifacts(id),
    retrieval_policy jsonb NOT NULL DEFAULT '{}'::jsonb,
    generation_policy jsonb NOT NULL DEFAULT '{}'::jsonb,
    evaluation_run_id uuid REFERENCES evaluation_runs(id),
    created_by_workflow_run_id uuid REFERENCES workflow_runs(id),
    promoted_at timestamptz,
    created_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (knowledge_base_id, alias, version)
);

CREATE UNIQUE INDEX releases_active_alias_idx
    ON releases (knowledge_base_id, alias)
    WHERE status = 'active';

CREATE TABLE rag_traces (
    id uuid PRIMARY KEY,
    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    knowledge_base_id uuid NOT NULL REFERENCES knowledge_bases(id),
    release_id uuid NOT NULL REFERENCES releases(id),
    request_kind text NOT NULL CHECK (request_kind IN ('retrieve', 'answer')),
    trace_id uuid NOT NULL UNIQUE,
    query_hash text NOT NULL,
    query_text text,
    response_text text,
    status text NOT NULL CHECK (status IN ('succeeded', 'failed')),
    latency_ms bigint NOT NULL CHECK (latency_ms >= 0),
    expires_at timestamptz NOT NULL,
    created_at timestamptz NOT NULL DEFAULT now()
);

CREATE INDEX rag_traces_project_created_idx
    ON rag_traces (project_id, created_at DESC);

CREATE INDEX rag_traces_expiry_idx ON rag_traces (expires_at);

CREATE TABLE rag_trace_evidence (
    id uuid PRIMARY KEY,
    rag_trace_id uuid NOT NULL REFERENCES rag_traces(id) ON DELETE CASCADE,
    rank integer NOT NULL CHECK (rank > 0),
    artifact_id uuid REFERENCES artifacts(id),
    external_ref text,
    score double precision,
    citation jsonb NOT NULL DEFAULT '{}'::jsonb,
    metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
    created_at timestamptz NOT NULL DEFAULT now(),
    UNIQUE (rag_trace_id, rank)
);

CREATE INDEX integration_instances_project_kind_idx
    ON integration_instances (project_id, kind, enabled);
CREATE INDEX knowledge_sources_project_idx ON knowledge_sources (project_id);
CREATE INDEX knowledge_bases_project_idx ON knowledge_bases (project_id);
CREATE INDEX artifacts_project_kind_idx ON artifacts (project_id, kind, created_at DESC);
CREATE INDEX datasets_project_idx ON datasets (project_id);
CREATE INDEX workflow_runs_project_status_idx
    ON workflow_runs (project_id, status, created_at DESC);
CREATE INDEX releases_project_idx ON releases (project_id, created_at DESC);

-- +goose Down

DROP TABLE rag_trace_evidence;
DROP TABLE rag_traces;
DROP TABLE releases;
DROP TABLE evaluation_results;
DROP TABLE evaluation_runs;
DROP TABLE evaluation_cases;
DROP TABLE evaluation_suites;
DROP TABLE dataset_versions;
DROP TABLE step_runs;
DROP TABLE workflow_runs;
DROP TABLE schedules;
DROP TABLE workflow_versions;
DROP TABLE workflow_definitions;
DROP TABLE datasets;
DROP TABLE artifact_lineage;
DROP TABLE artifacts;
DROP TABLE knowledge_base_sources;
DROP TABLE knowledge_bases;
DROP TABLE knowledge_sources;
DROP TABLE integration_instances;
DROP TABLE projects;