把三地 GPU 当成一个资源池：GKE 推理网关的内存感知全局路由

apiVersion: v1
kind: Service
metadata:
  name: llm-leader
  namespace: inference
spec:
  selector:
    app: llm-server
    inference-role: leader
  ports:
    - name: http
      port: 80
      targetPort: 30000
  type: ClusterIP

kubectl apply -f llm-leader-service.yaml

# 确认 Service 只选中了 leader，而不是 worker
kubectl -n inference get pods \
  -l app=llm-server,inference-role=leader \
  -o wide

kubectl -n inference get endpointslices \
  -l kubernetes.io/service-name=llm-leader \
  -o yaml

#!/usr/bin/env python3
import argparse
import concurrent.futures
import json
import os
import statistics
import time
import urllib.request


def percentile(values, p):
    values = sorted(values)
    if not values:
        return 0.0
    index = min(len(values) - 1, int((len(values) - 1) * p))
    return values[index]


def send(url, model, prompt, timeout):
    payload = {
        "model": model,
        "messages": [{"role": "user", "content": prompt}],
        "temperature": 0,
        "max_tokens": 128,
        "stream": False,
    }
    headers = {"Content-Type": "application/json"}
    api_key = os.getenv("API_KEY")
    if api_key:
        headers["Authorization"] = f"Bearer {api_key}"

    request = urllib.request.Request(
        url,
        data=json.dumps(payload).encode("utf-8"),
        headers=headers,
        method="POST",
    )

    started = time.perf_counter()
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            result = json.loads(response.read())
        latency = time.perf_counter() - started
        usage = result.get("usage", {})
        tokens = usage.get("total_tokens", 0)
        return True, latency, tokens, ""
    except Exception as exc:
        return False, time.perf_counter() - started, 0, str(exc)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--url", required=True)
    parser.add_argument("--model", required=True)
    parser.add_argument("--requests", type=int, default=100)
    parser.add_argument("--concurrency", type=int, default=10)
    parser.add_argument("--timeout", type=int, default=600)
    parser.add_argument("--prompt-file")
    args = parser.parse_args()

    prompt = "Explain why KV-cache pressure matters to LLM routing."
    if args.prompt_file:
        with open(args.prompt_file, "r", encoding="utf-8") as file:
            prompt = file.read()

    started = time.perf_counter()
    with concurrent.futures.ThreadPoolExecutor(
        max_workers=args.concurrency
    ) as executor:
        futures = [
            executor.submit(
                send, args.url, args.model, prompt, args.timeout
            )
            for _ in range(args.requests)
        ]
        results = [future.result() for future in futures]
    elapsed = time.perf_counter() - started

    successful = [item for item in results if item[0]]
    latencies = [item[1] for item in successful]
    total_tokens = sum(item[2] for item in successful)
    errors = [item[3] for item in results if not item[0]]

    report = {
        "requests": args.requests,
        "successful": len(successful),
        "success_rate": len(successful) / args.requests,
        "elapsed_seconds": round(elapsed, 3),
        "request_throughput_rps": round(len(successful) / elapsed, 3),
        "token_throughput_tps": round(total_tokens / elapsed, 3),
        "latency_mean_seconds": round(statistics.mean(latencies), 3)
        if latencies else None,
        "latency_p50_seconds": round(percentile(latencies, 0.50), 3),
        "latency_p95_seconds": round(percentile(latencies, 0.95), 3),
        "sample_errors": errors[:3],
    }
    print(json.dumps(report, indent=2))


if __name__ == "__main__":
    main()

export API_KEY='replace-me'

python3 global_probe.py \
  --url 'https://regional.example.com/v1/chat/completions' \
  --model 'your-model' \
  --requests 300 \
  --concurrency 30 \
  --timeout 600

python3 global_probe.py \
  --url 'https://global.example.com/v1/chat/completions' \
  --model 'your-model' \
  --requests 300 \
  --concurrency 30 \
  --timeout 600