Back to Repositories

Testing GitHub Issue Handler Error Resolution in OpenHands

This test suite focuses on error handling and retry mechanisms in the GitHub issue handler component of OpenHands. It verifies various error scenarios including rate limiting, network failures, and API responses while testing the robustness of the issue reference resolution system.

Test Coverage Overview

The test suite provides comprehensive coverage of error handling scenarios in the GitHub issue handler:

Non-existent issue reference handling
Rate limit error management
Network error handling
Successful issue reference resolution
Retry mechanism validation

Implementation Analysis

The testing approach utilizes pytest fixtures and mocking to simulate various API responses and error conditions. The implementation leverages unittest.mock for dependency injection and patches external service calls to ensure isolated testing of error handling logic.

Technical Details

Key technical components include:

pytest for test framework
unittest.mock for mocking
Custom fixtures for configuration
Mock LLM response classes
DotDict utility for response simulation

Best Practices Demonstrated

The test suite exemplifies several testing best practices:

Proper isolation of external dependencies
Comprehensive error scenario coverage
Clear test case organization
Effective use of fixtures and mocks
Validation of retry mechanisms and timeouts

all-hands-ai/openhands

tests/unit/resolver/test_issue_handler_error_handling.py

            
from unittest.mock import MagicMock, patch

import pytest
import requests
from litellm.exceptions import RateLimitError

from openhands.core.config import LLMConfig
from openhands.events.action.message import MessageAction
from openhands.llm.llm import LLM
from openhands.resolver.github_issue import GithubIssue
from openhands.resolver.issue_definitions import IssueHandler, PRHandler


@pytest.fixture(autouse=True)
def mock_logger(monkeypatch):
    # suppress logging of completion data to file
    mock_logger = MagicMock()
    monkeypatch.setattr('openhands.llm.debug_mixin.llm_prompt_logger', mock_logger)
    monkeypatch.setattr('openhands.llm.debug_mixin.llm_response_logger', mock_logger)
    return mock_logger


@pytest.fixture
def default_config():
    return LLMConfig(
        model='gpt-4o',
        api_key='test_key',
        num_retries=2,
        retry_min_wait=1,
        retry_max_wait=2,
    )


def test_handle_nonexistent_issue_reference():
    llm_config = LLMConfig(model='test', api_key='test')
    handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config)

    # Mock the requests.get to simulate a 404 error
    mock_response = MagicMock()
    mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
        '404 Client Error: Not Found'
    )

    with patch('requests.get', return_value=mock_response):
        # Call the method with a non-existent issue reference
        result = handler._PRHandler__get_context_from_external_issues_references(
            closing_issues=[],
            closing_issue_numbers=[],
            issue_body='This references #999999',  # Non-existent issue
            review_comments=[],
            review_threads=[],
            thread_comments=None,
        )

        # The method should return an empty list since the referenced issue couldn't be fetched
        assert result == []


def test_handle_rate_limit_error():
    llm_config = LLMConfig(model='test', api_key='test')
    handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config)

    # Mock the requests.get to simulate a rate limit error
    mock_response = MagicMock()
    mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
        '403 Client Error: Rate Limit Exceeded'
    )

    with patch('requests.get', return_value=mock_response):
        # Call the method with an issue reference
        result = handler._PRHandler__get_context_from_external_issues_references(
            closing_issues=[],
            closing_issue_numbers=[],
            issue_body='This references #123',
            review_comments=[],
            review_threads=[],
            thread_comments=None,
        )

        # The method should return an empty list since the request was rate limited
        assert result == []


def test_handle_network_error():
    llm_config = LLMConfig(model='test', api_key='test')
    handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config)

    # Mock the requests.get to simulate a network error
    with patch(
        'requests.get', side_effect=requests.exceptions.ConnectionError('Network Error')
    ):
        # Call the method with an issue reference
        result = handler._PRHandler__get_context_from_external_issues_references(
            closing_issues=[],
            closing_issue_numbers=[],
            issue_body='This references #123',
            review_comments=[],
            review_threads=[],
            thread_comments=None,
        )

        # The method should return an empty list since the network request failed
        assert result == []


def test_successful_issue_reference():
    llm_config = LLMConfig(model='test', api_key='test')
    handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config)

    # Mock a successful response
    mock_response = MagicMock()
    mock_response.raise_for_status.return_value = None
    mock_response.json.return_value = {'body': 'This is the referenced issue body'}

    with patch('requests.get', return_value=mock_response):
        # Call the method with an issue reference
        result = handler._PRHandler__get_context_from_external_issues_references(
            closing_issues=[],
            closing_issue_numbers=[],
            issue_body='This references #123',
            review_comments=[],
            review_threads=[],
            thread_comments=None,
        )

        # The method should return a list with the referenced issue body
        assert result == ['This is the referenced issue body']


class MockLLMResponse:
    """Mock LLM Response class to mimic the actual LLM response structure."""

    class Choice:
        class Message:
            def __init__(self, content):
                self.content = content

        def __init__(self, content):
            self.message = self.Message(content)

    def __init__(self, content):
        self.choices = [self.Choice(content)]


class DotDict(dict):
    """
    A dictionary that supports dot notation access.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for key, value in self.items():
            if isinstance(value, dict):
                self[key] = DotDict(value)
            elif isinstance(value, list):
                self[key] = [
                    DotDict(item) if isinstance(item, dict) else item for item in value
                ]

    def __getattr__(self, key):
        if key in self:
            return self[key]
        else:
            raise AttributeError(
                f"'{self.__class__.__name__}' object has no attribute '{key}'"
            )

    def __setattr__(self, key, value):
        self[key] = value

    def __delattr__(self, key):
        if key in self:
            del self[key]
        else:
            raise AttributeError(
                f"'{self.__class__.__name__}' object has no attribute '{key}'"
            )


@patch('openhands.llm.llm.litellm_completion')
def test_guess_success_rate_limit_wait_time(mock_litellm_completion, default_config):
    """Test that the retry mechanism in guess_success respects wait time between retries."""

    with patch('time.sleep') as mock_sleep:
        # Simulate a rate limit error followed by a successful response
        mock_litellm_completion.side_effect = [
            RateLimitError(
                'Rate limit exceeded', llm_provider='test_provider', model='test_model'
            ),
            DotDict(
                {
                    'choices': [
                        {
                            'message': {
                                'content': '--- success
true
--- explanation
Retry successful'
                            }
                        }
                    ]
                }
            ),
        ]

        llm = LLM(config=default_config)
        handler = IssueHandler('test-owner', 'test-repo', 'test-token', default_config)
        handler.llm = llm

        # Mock issue and history
        issue = GithubIssue(
            owner='test-owner',
            repo='test-repo',
            number=1,
            title='Test Issue',
            body='This is a test issue.',
            thread_comments=['Please improve error handling'],
        )
        history = [MessageAction(content='Fixed error handling.')]

        # Call guess_success
        success, _, explanation = handler.guess_success(issue, history)

        # Assertions
        assert success is True
        assert explanation == 'Retry successful'
        assert mock_litellm_completion.call_count == 2  # Two attempts made
        mock_sleep.assert_called_once()  # Sleep called once between retries

        # Validate wait time
        wait_time = mock_sleep.call_args[0][0]
        assert (
            default_config.retry_min_wait <= wait_time <= default_config.retry_max_wait
        ), f'Expected wait time between {default_config.retry_min_wait} and {default_config.retry_max_wait} seconds, but got {wait_time}'


@patch('openhands.llm.llm.litellm_completion')
def test_guess_success_exhausts_retries(mock_completion, default_config):
    """Test the retry mechanism in guess_success exhausts retries and raises an error."""
    # Simulate persistent rate limit errors by always raising RateLimitError
    mock_completion.side_effect = RateLimitError(
        'Rate limit exceeded', llm_provider='test_provider', model='test_model'
    )

    # Initialize LLM and handler
    llm = LLM(config=default_config)
    handler = PRHandler('test-owner', 'test-repo', 'test-token', default_config)
    handler.llm = llm

    # Mock issue and history
    issue = GithubIssue(
        owner='test-owner',
        repo='test-repo',
        number=1,
        title='Test Issue',
        body='This is a test issue.',
        thread_comments=['Please improve error handling'],
    )
    history = [MessageAction(content='Fixed error handling.')]

    # Call guess_success and expect it to raise an error after retries
    with pytest.raises(RateLimitError):
        handler.guess_success(issue, history)

    # Assertions
    assert (
        mock_completion.call_count == default_config.num_retries
    )  # Initial call + retries