LLM Structured Outputs: Stop Schema Drift in AI SaaS MVPs

+-----------------------------------------------------------------------------------------+
|                                THE NAIVE "JSON MODE" FAILURE TRAP                      |
+-----------------------------------------------------------------------------------------+
|                                                                                         |
|  [User Prompt] ---> [LLM + Basic JSON Mode] ---> [Raw String Output]                    |
|                                                             |                           |
|                                                             v                           |
|                                                  [JSON.parse() Passes]                  |
|                                                             |                           |
|                                     +-----------------------+-----------------------+   |
|                                     | Missing Keys / Hallucinated Enum Values       |   |
|                                     v                                               v   |
|                            [SQL NOT NULL Error]                          [Silent Data   |
|                            [500 API Crash]                                Ingestion]    |
+-----------------------------------------------------------------------------------------+

+-----------------------------------------------------------------------------------------+
|                           ENTERPRISE STRUCTURED OUTPUT PIPELINE                         |
+-----------------------------------------------------------------------------------------+
|                                                                                         |
|  1. Compile-Time Schema (Zod / Pydantic)                                                |
|     └─ Single Source of Truth for TypeScript/Python & JSON Schema                       |
|                             │                                                           |
|                             ▼                                                           |
|  2. Constrained Decoding (Provider Level: OpenAI / Anthropic / Local vLLM)              |
|     └─ Context-Free Grammar (CFG) / Logit Masking forces valid token generation         |
|                             │                                                           |
|                             ▼                                                           |
|  3. Runtime Boundary Validation & Error Recovery Loop                                   |
|     ├─ Strict Type Parsing & Sanitization                                               |
|     └─ Deterministic Retry Fallback (Max 1 attempt with error context)                  |
|                             │                                                           |
|                             ▼                                                           |
|  4. Type-Safe Persistence ([PostgreSQL Multi-Tenant DB])                                |
|                                                                                         |
+-----------------------------------------------------------------------------------------+

// src/server/ai/extractContractMetadata.ts
import OpenAI from "openai";
import { z } from "zod";
import { zodResponseFormat } from "openai/helpers/zod";

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  timeout: 15000, // 15-second strict timeout
  maxRetries: 2,
});

// 1. Define strict compile-time & runtime validation schema
export const ContractExtractionSchema = z.object({
  contractId: z.string().uuid(),
  counterparty: z.object({
    name: z.string().min(1),
    jurisdiction: z.enum(["US", "UK", "EU", "CA", "GLOBAL"]),
    vatOrTaxId: z.string().nullable(),
  }),
  financialTerms: z.object({
    totalValueCents: z.number().int().nonnegative(),
    currency: z.enum(["USD", "EUR", "GBP"]),
    paymentCadence: z.enum(["monthly", "quarterly", "annually", "one_time"]),
    autoRenew: z.boolean(),
  }),
  keyObligations: z.array(
    z.object({
      clauseTitle: z.string().max(100),
      riskLevel: z.enum(["low", "medium", "high", "critical"]),
      summary: z.string().max(500),
    })
  ).nonempty(),
  confidenceScore: z.number().min(0).max(1),
});

export type ContractExtraction = z.infer<typeof ContractExtractionSchema>;

interface ExtractionOptions {
  rawContractText: string;
  tenantId: string;
}

/**
 * Executes deterministic structured extraction with strict schema enforcement.
 */
export async function extractContractMetadata({
  rawContractText,
  tenantId,
}: ExtractionOptions): Promise<ContractExtraction> {
  try {
    const response = await openai.beta.chat.completions.parse({
      model: "gpt-4o-2024-08-06", // Must use model supporting strict structured outputs
      messages: [
        {
          role: "system",
          content:
            "You are an expert enterprise legal analyst. Extract the structured contractual metadata exactly matching the schema. If a value is unknown, use explicit null values where allowed.",
        },
        {
          role: "user",
          content: rawContractText,
        },
      ],
      // Enforce OpenAI Constrained Decoding via JSON Schema
      response_format: zodResponseFormat(
        ContractExtractionSchema,
        "contract_extraction"
      ),
      temperature: 0.0, // Zero temperature for maximum determinism
    });

    const parsedData = response.choices[0]?.message.parsed;
    const refusal = response.choices[0]?.message.refusal;

    if (refusal) {
      throw new Error(`Model safety refusal encountered: ${refusal}`);
    }

    if (!parsedData) {
      throw new Error("LLM returned an empty parsed response payload.");
    }

    // 2. Perform second-tier runtime boundary validation
    const validatedResult = ContractExtractionSchema.parse(parsedData);
    return validatedResult;
  } catch (error) {
    // Log structured error for observability and trigger fallback or alerting
    console.error("[AI_EXTRACTION_FAILURE]", {
      tenantId,
      error: error instanceof Error ? error.message : String(error),
    });
    throw new Error("Failed to extract structured contract data with type safety.");
  }
}

-- Always inject tenant_id from the authenticated user session, NOT the LLM payload
INSERT INTO contract_extractions (
    id,
    tenant_id,
    contract_id,
    counterparty_name,
    jurisdiction,
    total_value_cents,
    currency,
    obligations_payload,
    confidence_score,
    created_at
)
VALUES (
    gen_random_uuid(),
    $1, -- Secure session tenant_id (e.g., from auth context)
    $2, -- contractId validated by Zod
    $3, -- counterparty.name
    $4, -- counterparty.jurisdiction
    $5, -- financialTerms.totalValueCents
    $6, -- financialTerms.currency
    $7::jsonb, -- keyObligations array stored as queryable JSONB
    $8, -- confidenceScore
    NOW()
);