#!/usr/bin/env bash
set -eo pipefail

# Mount shared memory to prevent NCCL thread locks
mount -o remount,size=128g /dev/shm

# Execute vLLM engine with specialized MoE execution flags
python3 -m vllm.entrypoints.openai.api_server \
    --model /opt/ml/model \
    --tensor-parallel-size 8 \
    --pipeline-parallel-size 1 \
    --load-format safetensors \
    --gpu-memory-utilization 0.95 \
    --max-num-seqs 256 \
    --trust-remote-code \
    --enforce-eager \
    --quantization fp8 \
    --port 8080

import boto3
import json

# Setup the SageMaker runtime client
runtime = boto3.client('sagemaker-runtime', region_name='us-east-1')

# Define target test payload requesting reasoning chain outputs
payload = {
    "model": "deepseek-r1",
    "messages": [
        {"role": "user", "content": "Prove that the square root of any prime number is irrational."}
    ],
    "temperature": 0.1,
    "max_tokens": 1024
}

# Run execution
response = runtime.invoke_endpoint(
    EndpointName='deepseek-r1-fp8-endpoint',
    ContentType='application/json',
    Body=json.dumps(payload)
)

# Render results
result = json.loads(response['Body'].read().decode('utf-8'))
print(json.dumps(result, indent=2))

import time
import logging
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("MLOpsMiddleware")

app = FastAPI()

@app.middleware("http")
async def audit_llm_metrics_middleware(request: Request, call_next):
    """
    MLOps middleware to track LLM latency metrics, token consumption rates,
    and prevent service crashes due to malformed payloads.
    """
    start_time = time.time()
    
    # Run request payload validation checks
    if request.method == "POST":
        try:
            body = await request.json()
            # Enforce strict output limits
            if body.get("max_tokens", 0) > 4096:
                return JSONResponse(
                    status_code=400,
                    content={"error": "The maximum tokens requested cannot exceed 4096 to protect system stability."}
                )
        except Exception:
            pass  # Fail-safe pass-through for non-JSON payloads
            
    response = await call_next(request)
    
    # Record and log operational metrics
    duration = time.time() - start_time
    logger.info(f"Route: {request.url.path} | Duration: {duration:.4f}s | Status: {response.status_code}")
    
    return response