Log in
Log in
Log in
Book a Demo
Get Started
734.8K 537.1K 9K
Back to Repositories

Testing File Edit Operations and Diff Generation in OpenHands

A comprehensive test suite for validating file editing functionality in the EventStreamRuntime of OpenHands. This test file focuses on verifying file creation, modification, and observation handling for both small and large files, ensuring robust edit operations across different scenarios.

Test Coverage Overview

The test suite provides extensive coverage of file editing operations with multiple test cases:
  • Creating new files from scratch
  • Modifying existing files with content updates
  • Handling large file edits with specific line ranges
  • Validating edit observations and diff generation
  • Testing multiple simultaneous edits

Implementation Analysis

The testing approach utilizes pytest fixtures and skipif decorators for conditional test execution. It implements a systematic pattern of creating files, performing edits, and validating both the edit process and resulting content through FileEditAction and FileReadAction operations.

Each test case follows a clear structure of setup, action execution, and assertion verification.

Technical Details

Key technical components include:
  • pytest framework for test organization
  • Custom fixtures: temp_dir, runtime_cls, run_as_openhands
  • FileEditAction and FileReadAction classes
  • diff utility for content comparison
  • Conditional CI execution controls

Best Practices Demonstrated

The test suite exemplifies several testing best practices:
  • Proper test isolation using fixtures
  • Comprehensive error handling and cleanup
  • Clear test case organization
  • Detailed assertion messages
  • Edge case coverage for large files and multiple edits

all-hands-ai/openhands

tests/runtime/test_edit.py

            
"""Edit-related tests for the EventStreamRuntime."""

import os

import pytest
from conftest import TEST_IN_CI, _close_test_runtime, _load_runtime
from openhands_aci.utils.diff import get_diff

from openhands.core.logger import openhands_logger as logger
from openhands.events.action import FileEditAction, FileReadAction
from openhands.events.observation import FileEditObservation

ORGINAL = """from flask import Flask
app = Flask(__name__)

@app.route('/')
def index():
    numbers = list(range(1, 11))
    return str(numbers)

if __name__ == '__main__':
    app.run(port=5000)
"""


@pytest.mark.skipif(
    TEST_IN_CI != 'True',
    reason='This test requires LLM to run.',
)
def test_edit_from_scratch(temp_dir, runtime_cls, run_as_openhands):
    runtime = _load_runtime(temp_dir, runtime_cls, run_as_openhands)
    try:
        action = FileEditAction(
            content=ORGINAL,
            start=-1,
            path=os.path.join('/workspace', 'app.py'),
        )
        logger.info(action, extra={'msg_type': 'ACTION'})
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})

        assert isinstance(
            obs, FileEditObservation
        ), 'The observation should be a FileEditObservation.'

        action = FileReadAction(
            path=os.path.join('/workspace', 'app.py'),
        )
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
        assert obs.content.strip() == ORGINAL.strip()

    finally:
        _close_test_runtime(runtime)


EDIT = """# above stays the same
@app.route('/')
def index():
    numbers = list(range(1, 11))
    return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
# below stays the same
"""


@pytest.mark.skipif(
    TEST_IN_CI != 'True',
    reason='This test requires LLM to run.',
)
def test_edit(temp_dir, runtime_cls, run_as_openhands):
    runtime = _load_runtime(temp_dir, runtime_cls, run_as_openhands)
    try:
        action = FileEditAction(
            content=ORGINAL,
            path=os.path.join('/workspace', 'app.py'),
        )
        logger.info(action, extra={'msg_type': 'ACTION'})
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})

        assert isinstance(
            obs, FileEditObservation
        ), 'The observation should be a FileEditObservation.'

        action = FileReadAction(
            path=os.path.join('/workspace', 'app.py'),
        )
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
        assert obs.content.strip() == ORGINAL.strip()

        action = FileEditAction(
            content=EDIT,
            path=os.path.join('/workspace', 'app.py'),
        )
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
        assert (
            obs.content.strip()
            == (
                '--- /workspace/app.py
'
                '+++ /workspace/app.py
'
                '@@ -4,7 +4,7 @@
'
                " @app.route('/')
"
                ' def index():
'
                '     numbers = list(range(1, 11))
'
                '-    return str(numbers)
'
                "+    return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
"
                '
'
                " if __name__ == '__main__':
"
                '     app.run(port=5000)
'
            ).strip()
        )
    finally:
        _close_test_runtime(runtime)


ORIGINAL_LONG = '
'.join([f'This is line {i}' for i in range(1, 1000)])
EDIT_LONG = """
This is line 100 + 10
This is line 101 + 10
"""


@pytest.mark.skipif(
    TEST_IN_CI != 'True',
    reason='This test requires LLM to run.',
)
def test_edit_long_file(temp_dir, runtime_cls, run_as_openhands):
    runtime = _load_runtime(temp_dir, runtime_cls, run_as_openhands)
    try:
        action = FileEditAction(
            content=ORIGINAL_LONG,
            path=os.path.join('/workspace', 'app.py'),
            start=-1,
        )
        logger.info(action, extra={'msg_type': 'ACTION'})
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})

        assert isinstance(
            obs, FileEditObservation
        ), 'The observation should be a FileEditObservation.'

        action = FileReadAction(
            path=os.path.join('/workspace', 'app.py'),
        )
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
        assert obs.content.strip() == ORIGINAL_LONG.strip()

        action = FileEditAction(
            content=EDIT_LONG,
            path=os.path.join('/workspace', 'app.py'),
            start=100,
            end=200,
        )
        obs = runtime.run_action(action)
        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
        assert (
            obs.content.strip()
            == (
                '--- /workspace/app.py
'
                '+++ /workspace/app.py
'
                '@@ -97,8 +97,8 @@
'
                ' This is line 97
'
                ' This is line 98
'
                ' This is line 99
'
                '-This is line 100
'
                '-This is line 101
'
                '+This is line 100 + 10
'
                '+This is line 101 + 10
'
                ' This is line 102
'
                ' This is line 103
'
                ' This is line 104
'
            ).strip()
        )
    finally:
        _close_test_runtime(runtime)


# ======================================================================================
# Test FileEditObservation (things that are displayed to the agent)
# ======================================================================================


def test_edit_obs_insert_only():
    EDIT_LONG_INSERT_ONLY = (
        '
'.join([f'This is line {i}' for i in range(1, 100)])
        + EDIT_LONG
        + '
'.join([f'This is line {i}' for i in range(100, 1000)])
    )

    diff = get_diff(ORIGINAL_LONG, EDIT_LONG_INSERT_ONLY, '/workspace/app.py')
    obs = FileEditObservation(
        content=diff,
        path='/workspace/app.py',
        prev_exist=True,
        old_content=ORIGINAL_LONG,
        new_content=EDIT_LONG_INSERT_ONLY,
    )
    assert (
        str(obs).strip()
        == """
[Existing file /workspace/app.py is edited with 1 changes.]
[begin of edit 1 / 1]
(content before edit)
  98|This is line 98
  99|This is line 99
 100|This is line 100
 101|This is line 101
(content after edit)
  98|This is line 98
  99|This is line 99
+100|This is line 100 + 10
+101|This is line 101 + 10
 102|This is line 100
 103|This is line 101
[end of edit 1 / 1]
""".strip()
    )


def test_edit_obs_replace():
    _new_content = (
        '
'.join([f'This is line {i}' for i in range(1, 100)])
        + EDIT_LONG
        + '
'.join([f'This is line {i}' for i in range(102, 1000)])
    )

    diff = get_diff(ORIGINAL_LONG, _new_content, '/workspace/app.py')
    obs = FileEditObservation(
        content=diff,
        path='/workspace/app.py',
        prev_exist=True,
        old_content=ORIGINAL_LONG,
        new_content=_new_content,
    )
    print(str(obs))
    assert (
        str(obs).strip()
        == """
[Existing file /workspace/app.py is edited with 1 changes.]
[begin of edit 1 / 1]
(content before edit)
  98|This is line 98
  99|This is line 99
-100|This is line 100
-101|This is line 101
 102|This is line 102
 103|This is line 103
(content after edit)
  98|This is line 98
  99|This is line 99
+100|This is line 100 + 10
+101|This is line 101 + 10
 102|This is line 102
 103|This is line 103
[end of edit 1 / 1]
""".strip()
    )


def test_edit_obs_replace_with_empty_line():
    _new_content = (
        '
'.join([f'This is line {i}' for i in range(1, 100)])
        + '
'
        + EDIT_LONG
        + '
'.join([f'This is line {i}' for i in range(102, 1000)])
    )

    diff = get_diff(ORIGINAL_LONG, _new_content, '/workspace/app.py')
    obs = FileEditObservation(
        content=diff,
        path='/workspace/app.py',
        prev_exist=True,
        old_content=ORIGINAL_LONG,
        new_content=_new_content,
    )
    print(str(obs))
    assert (
        str(obs).strip()
        == """
[Existing file /workspace/app.py is edited with 1 changes.]
[begin of edit 1 / 1]
(content before edit)
  98|This is line 98
  99|This is line 99
-100|This is line 100
-101|This is line 101
 102|This is line 102
 103|This is line 103
(content after edit)
  98|This is line 98
  99|This is line 99
+100|
+101|This is line 100 + 10
+102|This is line 101 + 10
 103|This is line 102
 104|This is line 103
[end of edit 1 / 1]
""".strip()
    )


def test_edit_obs_multiple_edits():
    _new_content = (
        '
'.join([f'This is line {i}' for i in range(1, 50)])
        + '
balabala
'
        + '
'.join([f'This is line {i}' for i in range(50, 100)])
        + EDIT_LONG
        + '
'.join([f'This is line {i}' for i in range(102, 1000)])
    )

    diff = get_diff(ORIGINAL_LONG, _new_content, '/workspace/app.py')
    obs = FileEditObservation(
        content=diff,
        path='/workspace/app.py',
        prev_exist=True,
        old_content=ORIGINAL_LONG,
        new_content=_new_content,
    )
    assert (
        str(obs).strip()
        == """
[Existing file /workspace/app.py is edited with 2 changes.]
[begin of edit 1 / 2]
(content before edit)
 48|This is line 48
 49|This is line 49
 50|This is line 50
 51|This is line 51
(content after edit)
 48|This is line 48
 49|This is line 49
+50|balabala
 51|This is line 50
 52|This is line 51
[end of edit 1 / 2]
-------------------------
[begin of edit 2 / 2]
(content before edit)
  98|This is line 98
  99|This is line 99
-100|This is line 100
-101|This is line 101
 102|This is line 102
 103|This is line 103
(content after edit)
  99|This is line 98
 100|This is line 99
+101|This is line 100 + 10
+102|This is line 101 + 10
 103|This is line 102
 104|This is line 103
[end of edit 2 / 2]
""".strip()
    )


def test_edit_visualize_failed_edit():
    _new_content = (
        '
'.join([f'This is line {i}' for i in range(1, 50)])
        + '
balabala
'
        + '
'.join([f'This is line {i}' for i in range(50, 100)])
        + EDIT_LONG
        + '
'.join([f'This is line {i}' for i in range(102, 1000)])
    )

    diff = get_diff(ORIGINAL_LONG, _new_content, '/workspace/app.py')
    obs = FileEditObservation(
        content=diff,
        path='/workspace/app.py',
        prev_exist=True,
        old_content=ORIGINAL_LONG,
        new_content=_new_content,
    )
    assert (
        obs.visualize_diff(change_applied=False).strip()
        == """
[Changes are NOT applied to /workspace/app.py - Here's how the file looks like if changes are applied.]
[begin of ATTEMPTED edit 1 / 2]
(content before ATTEMPTED edit)
 48|This is line 48
 49|This is line 49
 50|This is line 50
 51|This is line 51
(content after ATTEMPTED edit)
 48|This is line 48
 49|This is line 49
+50|balabala
 51|This is line 50
 52|This is line 51
[end of ATTEMPTED edit 1 / 2]
-------------------------
[begin of ATTEMPTED edit 2 / 2]
(content before ATTEMPTED edit)
  98|This is line 98
  99|This is line 99
-100|This is line 100
-101|This is line 101
 102|This is line 102
 103|This is line 103
(content after ATTEMPTED edit)
  99|This is line 98
 100|This is line 99
+101|This is line 100 + 10
+102|This is line 101 + 10
 103|This is line 102
 104|This is line 103
[end of ATTEMPTED edit 2 / 2]
""".strip()
    )