使用阿里云服务网格 ASM LLMProxy 插件保障大模型用户数据安全

apiVersion: v1
kind: ServiceAccount
metadata:
  name: sleep
---
apiVersion: v1
kind: Service
metadata:
  name: sleep
  labels:
    app: sleep
    service: sleep
spec:
  ports:
  - port: 80
    name: http
  selector:
    app: sleep
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: sleep
spec:
  replicas: 1
  selector:
    matchLabels:
      app: sleep
  template:
    metadata:
      labels:
        app: sleep
    spec:
      terminationGracePeriodSeconds: 0
      serviceAccountName: sleep
      containers:
      - name: sleep
        image: registry.cn-hangzhou.aliyuncs.com/acs/curl:8.1.2
        command: ["/bin/sleep", "infinity"]
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /etc/sleep/tls
          name: secret-volume
      volumes:
      - name: secret-volume
        secret:
          secretName: sleep-secret
          optional: true
---

apiVersion: networking.istio.io/v1beta1
kind: ServiceEntry
metadata:
  name: dashscope
  namespace: default
spec:
  hosts:
    - dashscope.aliyuncs.com
  ports:
    - name: http-port
      number: 80
      protocol: HTTP
      targetPort: 443    # 和Destination配合使用，用于将HTTP协议升级为HTTPS
    - name: https-port
      number: 443
      protocol: HTTPS
  resolution: DNS

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: dashscope
  namespace: default
spec:
  host: dashscope.aliyuncs.com
  trafficPolicy:
    portLevelSettings:
    - port:
        number: 80
      tls:
        mode: SIMPLE

kubectl exec ${sleep pod name} -- curl -v 'http://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions' \
--header 'Authorization: Bearer ${dashscope API_KEY}' \
--header 'Content-Type: application/json' \
--header 'user: test' \
--data '{
    "model": "qwen-turbo",
    "messages": [
        {"role": "user", "content": "你是谁"}
    ],
    "stream": false
}'

{"choices":[{"message":{"role":"assistant","content":"我是来自阿里云的大规模语言模型，我叫通义千问。"},"finish_reason":"stop","index":0,"logprobs":null}],"object":"chat.completion","usage":{"prompt_tokens":10,"completion_tokens":16,"total_tokens":26},"created":xxxxxxxx,"system_fingerprint":null,"model":"qwen-turbo","id":"xxxxxxxxxxxxxxxxxx"}

apiVersion: extensions.istio.io/v1alpha1
kind: WasmPlugin
metadata:
  name: asm-llm-proxy
  namespace: default
spec:
  imagePullPolicy: Always
  phase: AUTHN
  selector:
    matchLabels:
      app: sleep
  url: registry-cn-hangzhou.ack.aliyuncs.com/test/asm-llm-proxy:v0.2
  pluginConfig:
    api_key: ${dashscope的API_KEY}
    deny_patterns:
    - .*账号.*     # 禁止包含“账号”两个字的message被发往外部大模型
    hosts:
    - dashscope.aliyuncs.com    # 该插件只对host为dashscope.aliyuncs.com的请求生效
    intelligent_guard:   # 配置一个私有LLM服务，对请求进行敏感信息验证。
      # 本文为了验证方便，依然调用灵积服务来对请求进行验证。
      api_key: ${私有LLM服务的API_KEY}
      host: dashscope.aliyuncs.com
      model: qwen-turbo
      path: /compatible-mode/v1/chat/completions
      port: 80  # serviceentry中的HTTP端口

kubectl exec ${sleep pod name} -- curl 'http://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
    "model": "qwen-turbo",
    "messages": [
        {"role": "user", "content": "你是谁"}
    ],
    "stream": false
}'

{"choices":[{"message":{"role":"assistant","content":"我是来自阿里云的大规模语言模型，我叫通义千问。"},"finish_reason":"stop","index":0,"logprobs":null}],"object":"chat.completion","usage":{"prompt_tokens":10,"completion_tokens":16,"total_tokens":26},"created":xxxxxxx,"system_fingerprint":null,"model":"qwen-turbo","id":"xxxxxxxxx"}

kubectl exec ${sleep pod name} -- curl 'http://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
    "model": "qwen-turbo",
    "messages": [
        {"role": "user", "content": "我喜欢吃豆沙粽子，我的QQ账号是1111111"}
    ],
    "stream": false
}'

request was denied by asm llm proxy

kubectl exec ${sleep pod name} -- curl 'http://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
    "model": "qwen-turbo",
    "messages": [
        {"role": "user", "content": "我们公司将会在9月10日举行内部高级别会议，会议主题是如何更好的服务客户，请给我一份会议开场白。"}
    ],
    "stream": false
}'