Back to Repositories

Testing OpenAI Vision API Integration in FastChat

This test suite validates the OpenAI-compatible vision API server implementation in FastChat, focusing on image processing capabilities and chat completions with both text and image inputs.

Test Coverage Overview

The test suite provides comprehensive coverage of OpenAI vision API functionality:

Model listing and availability checks
Chat completions with text-only inputs
Image processing with URL and base64 encoded inputs
Streaming chat completion responses
API endpoint validation via curl requests

Implementation Analysis

The testing approach implements multiple validation scenarios using the OpenAI Python client library. Key patterns include:

Image encoding and format conversion
Multiple message content types handling
Streaming and non-streaming response processing
Direct API endpoint testing

Technical Details

Testing infrastructure includes:

OpenAI Python SDK integration
PIL for image processing
Base64 encoding utilities
HTTP request handling
Local server setup (port 8000)
Custom test server launch script

Best Practices Demonstrated

The test suite exemplifies several testing best practices:

Modular test function organization
Comprehensive API endpoint coverage
Multiple input format validation
Error handling and edge cases
Clear test output formatting

lm-sys/fastchat

tests/test_openai_vision_api.py

            
"""
Test the OpenAI compatible server

Launch:
python3 launch_openai_api_test_server.py --multimodal
"""

import openai

from fastchat.utils import run_cmd

openai.api_key = "EMPTY"  # Not support yet
openai.base_url = "http://localhost:8000/v1/"


def encode_image(image):
    import base64
    from io import BytesIO
    import requests

    from PIL import Image

    if image.startswith("http://") or image.startswith("https://"):
        response = requests.get(image)
        image = Image.open(BytesIO(response.content)).convert("RGB")
    else:
        image = Image.open(image).convert("RGB")

    buffered = BytesIO()
    image.save(buffered, format="PNG")
    img_b64_str = base64.b64encode(buffered.getvalue()).decode("utf-8")

    return img_b64_str


def test_list_models():
    model_list = openai.models.list()
    names = [x.id for x in model_list.data]
    return names


def test_chat_completion(model):
    image_url = "https://picsum.photos/seed/picsum/1024/1024"
    base64_image_url = f"data:image/jpeg;base64,{encode_image(image_url)}"

    # No Image
    completion = openai.chat.completions.create(
        model=model,
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Tell me about alpacas."},
                ],
            }
        ],
        temperature=0,
    )
    print(completion.choices[0].message.content)
    print("=" * 25)

    # Image using url link
    completion = openai.chat.completions.create(
        model=model,
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "What’s in this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                ],
            }
        ],
        temperature=0,
    )
    print(completion.choices[0].message.content)
    print("=" * 25)

    # Image using base64 image url
    completion = openai.chat.completions.create(
        model=model,
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "What’s in this image?"},
                    {"type": "image_url", "image_url": {"url": base64_image_url}},
                ],
            }
        ],
        temperature=0,
    )
    print(completion.choices[0].message.content)
    print("=" * 25)


def test_chat_completion_stream(model):
    image_url = "https://picsum.photos/seed/picsum/1024/1024"

    messages = [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What’s in this image?"},
                {"type": "image_url", "image_url": {"url": image_url}},
            ],
        }
    ]
    res = openai.chat.completions.create(
        model=model, messages=messages, stream=True, temperature=0
    )
    for chunk in res:
        try:
            content = chunk.choices[0].delta.content
            if content is None:
                content = ""
        except Exception as e:
            content = chunk.choices[0].delta.get("content", "")
        print(content, end="", flush=True)
    print()


def test_openai_curl():
    run_cmd(
        """curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llava-v1.5-7b",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "text",
            "text": "What’s in this image?"
          },
          {
            "type": "image_url",
            "image_url": {
              "url": "https://picsum.photos/seed/picsum/1024/1024"
            }
          }
        ]
      }
    ],
    "max_tokens": 300
  }'
            """
    )

    print()


if __name__ == "__main__":
    models = test_list_models()
    print(f"models: {models}")

    for model in models:
        print(f"===== Test {model} ======")
        test_chat_completion(model)
        test_chat_completion_stream(model)
    test_openai_curl()