Back to Repositories

Testing CLI Agent Configuration and Execution in gpt-engineer

This test suite validates the CLI Agent functionality in gpt-engineer, focusing on initialization and improvement capabilities across different configuration modes. The tests ensure proper file handling, code generation, and execution environment setup.

Test Coverage Overview

The test suite provides comprehensive coverage of the CliAgent class functionality.

Key areas tested include:

Standard configuration initialization
Lite configuration setup
Clarified generation mode
Code improvement capabilities

Edge cases cover different input handling and file operations across various configuration modes.

Implementation Analysis

The testing approach utilizes pytest fixtures and mocking to isolate components and simulate AI responses. The implementation leverages monkeypatch for input simulation and temporary directory management for file operations testing.

Technical patterns include:

Mock AI response handling
File system operations
Environment configuration testing
Code generation validation

Technical Details

Testing tools and configuration:

pytest for test framework
tempfile for temporary directory management
MockAI for AI response simulation
DiskMemory for storage operations
DiskExecutionEnv for execution environment

Best Practices Demonstrated

The test suite exemplifies robust testing practices through isolation of components and comprehensive validation.

Notable practices include:

Proper test isolation using fixtures
Comprehensive mocking of external dependencies
Clear test case organization
Validation of both happy path and edge cases

gpt-engineer-org/gpt-engineer

tests/applications/cli/test_cli_agent.py

            
import os
import tempfile

import pytest

from langchain.schema import AIMessage

from gpt_engineer.applications.cli.cli_agent import CliAgent
from gpt_engineer.core.default.disk_execution_env import DiskExecutionEnv
from gpt_engineer.core.default.disk_memory import DiskMemory

# from gpt_engineer.core.default.git_version_manager import GitVersionManager
from gpt_engineer.core.default.paths import ENTRYPOINT_FILE, memory_path
from gpt_engineer.core.files_dict import FilesDict
from gpt_engineer.core.prompt import Prompt
from gpt_engineer.tools.custom_steps import clarified_gen, lite_gen
from tests.mock_ai import MockAI


def test_init_standard_config(monkeypatch):
    monkeypatch.setattr("builtins.input", lambda _: "y")
    temp_dir = tempfile.mkdtemp()
    memory = DiskMemory(memory_path(temp_dir))
    execution_env = DiskExecutionEnv()
    mock_ai = MockAI(
        [
            AIMessage(
                "hello_world.py
```
with open('output.txt', 'w') as file:
    file.write('Hello World!')
```"
            ),
            AIMessage("```run.sh
python3 hello_world.py
```"),
        ],
    )
    cli_agent = CliAgent.with_default_config(memory, execution_env, ai=mock_ai)
    outfile = "output.txt"
    os.path.join(temp_dir, outfile)
    code = cli_agent.init(
        Prompt(
            f"Make a program that prints 'Hello World!' to a file called '{outfile}'"
        )
    )

    env = DiskExecutionEnv()
    env.upload(code).run(f"bash {ENTRYPOINT_FILE}")
    code = env.download()

    assert outfile in code
    assert code[outfile] == "Hello World!"


def test_init_lite_config(monkeypatch):
    monkeypatch.setattr("builtins.input", lambda _: "y")
    temp_dir = tempfile.mkdtemp()
    memory = DiskMemory(memory_path(temp_dir))
    # version_manager = GitVersionManager(temp_dir)
    execution_env = DiskExecutionEnv()
    mock_ai = MockAI(
        [
            AIMessage(
                "hello_world.py
```
with open('output.txt', 'w') as file:
    file.write('Hello World!')
```"
            ),
            AIMessage("```run.sh
python3 hello_world.py
```"),
        ],
    )
    cli_agent = CliAgent.with_default_config(
        memory, execution_env, ai=mock_ai, code_gen_fn=lite_gen
    )
    outfile = "output.txt"
    os.path.join(temp_dir, outfile)
    code = cli_agent.init(
        Prompt(
            f"Make a program that prints 'Hello World!' to a file called '{outfile}'"
        )
    )

    env = DiskExecutionEnv()
    env.upload(code).run(f"bash {ENTRYPOINT_FILE}")
    code = env.download()

    assert outfile in code
    assert code[outfile].strip() == "Hello World!"


def test_init_clarified_gen_config(monkeypatch):
    monkeypatch.setattr("builtins.input", lambda _: "y")
    temp_dir = tempfile.mkdtemp()
    memory = DiskMemory(memory_path(temp_dir))
    execution_env = DiskExecutionEnv()
    mock_ai = MockAI(
        [
            AIMessage("nothing to clarify"),
            AIMessage(
                "hello_world.py
```
with open('output.txt', 'w') as file:
    file.write('Hello World!')
```"
            ),
            AIMessage("```run.sh
python3 hello_world.py
```"),
        ],
    )
    cli_agent = CliAgent.with_default_config(
        memory, execution_env, ai=mock_ai, code_gen_fn=clarified_gen
    )
    outfile = "output.txt"
    code = cli_agent.init(
        Prompt(
            f"Make a program that prints 'Hello World!' to a file called '{outfile} either using python or javascript'"
        )
    )

    env = DiskExecutionEnv()
    env.upload(code).run(f"bash {ENTRYPOINT_FILE}")
    code = env.download()

    assert outfile in code
    assert code[outfile].strip() == "Hello World!"


def test_improve_standard_config(monkeypatch):
    monkeypatch.setattr("builtins.input", lambda _: "y")
    temp_dir = tempfile.mkdtemp()
    code = FilesDict(
        {
            "main.py": "def write_hello_world_to_file(filename):
    \"\"\"
    Writes 'Hello World!' to the specified file.
    
    :param filename: The name of the file to write to.
    \"\"\"
    with open(filename, 'w') as file:
        file.write('Hello World!')

if __name__ == \"__main__\":
    output_filename = 'output.txt'
    write_hello_world_to_file(output_filename)",
            "requirements.txt": "# No dependencies required",
            "run.sh": "python3 main.py
",
        }
    )
    memory = DiskMemory(memory_path(temp_dir))
    # version_manager = GitVersionManager(temp_dir)
    execution_env = DiskExecutionEnv()
    mock_ai = MockAI(
        [
            AIMessage(
                "```diff
--- main.py
+++ main.py
@@ -7,3 +7,3 @@
     with open(filename, 'w') as file:
-        file.write('Hello World!')
+        file.write('!dlroW olleH')
```"
            )
        ]
    )
    cli_agent = CliAgent.with_default_config(memory, execution_env, ai=mock_ai)

    code = cli_agent.improve(
        code,
        Prompt(
            "Change the program so that it prints '!dlroW olleH' instead of 'Hello World!'"
        ),
    )

    env = DiskExecutionEnv()
    env.upload(code).run(f"bash {ENTRYPOINT_FILE}")
    code = env.download()

    outfile = "output.txt"
    assert outfile in code
    assert code[outfile] == "!dlroW olleH"


if __name__ == "__main__":
    pytest.main()