Back to Repositories

Testing Scientist Result Class Observation Handling in github/scientist

This test suite validates the Scientist::Result class functionality in Ruby, focusing on observation handling, matching logic, and experiment context management. The tests ensure proper immutability, comparison mechanisms, and error handling for scientific experiments.

Test Coverage Overview

The test suite provides comprehensive coverage of the Scientist::Result class functionality.

Key areas tested include:

Result object immutability
Observation evaluation and matching
Control-only observation scenarios
Custom comparison implementations
Mismatch handling and ignore logic

Implementation Analysis

The testing approach uses RSpec’s describe/it blocks for structured test organization. Tests leverage before blocks for setup and multiple assertion patterns to verify behavior.

Implementation highlights:

Block-based test definitions
Observation creation with dynamic values
Custom comparison block usage
Context management verification

Technical Details

Testing tools and configuration:

RSpec testing framework
Scientist::Experiment class integration
Scientist::Observation object manipulation
Custom comparison blocks
Context hash management

Best Practices Demonstrated

The test suite exemplifies several testing best practices.

Notable practices include:

Isolated test cases with clear purposes
Comprehensive edge case coverage
Proper setup and teardown management
Clear test naming conventions
Effective use of assertion variations

github/scientist

test/scientist/result_test.rb

            
describe Scientist::Result do
  before do
    @experiment = Scientist::Experiment.new "experiment"
  end

  it "is immutable" do
    control = Scientist::Observation.new("control", @experiment)
    candidate = Scientist::Observation.new("candidate", @experiment)

    result = Scientist::Result.new @experiment, [control, candidate], control
    assert result.frozen?
  end

  it "evaluates its observations" do
    a = Scientist::Observation.new("a", @experiment) { 1 }
    b = Scientist::Observation.new("b", @experiment) { 1 }

    assert a.equivalent_to?(b)

    result = Scientist::Result.new @experiment, [a, b], a
    assert result.matched?
    refute result.mismatched?
    assert_equal [], result.mismatched

    x = Scientist::Observation.new("x", @experiment) { 1 }
    y = Scientist::Observation.new("y", @experiment) { 2 }
    z = Scientist::Observation.new("z", @experiment) { 3 }

    result = Scientist::Result.new @experiment, [x, y, z], x
    refute result.matched?
    assert result.mismatched?
    assert_equal [y, z], result.mismatched
  end

  it "has no mismatches if there is only a control observation" do
    a = Scientist::Observation.new("a", @experiment) { 1 }
    result = Scientist::Result.new @experiment, [a], a
    assert result.matched?
  end

  it "evaluates observations using the experiment's compare block" do
    a = Scientist::Observation.new("a", @experiment) { "1" }
    b = Scientist::Observation.new("b", @experiment) { 1 }

    @experiment.compare { |x, y| x == y.to_s }

    result = Scientist::Result.new @experiment, [a, b], a

    assert result.matched?, result.mismatched
  end

  it "does not ignore any mismatches when nothing's ignored" do
    x = Scientist::Observation.new("x", @experiment) { 1 }
    y = Scientist::Observation.new("y", @experiment) { 2 }

    result = Scientist::Result.new @experiment, [x, y], x

    assert result.mismatched?
    refute result.ignored?
  end

  it "uses the experiment's ignore block to ignore mismatched observations" do
    x = Scientist::Observation.new("x", @experiment) { 1 }
    y = Scientist::Observation.new("y", @experiment) { 2 }
    called = false
    @experiment.ignore { called = true }

    result = Scientist::Result.new @experiment, [x, y], x

    refute result.mismatched?
    refute result.matched?
    assert result.ignored?
    assert_equal [], result.mismatched
    assert_equal [y], result.ignored
    assert called
  end

  it "partitions observations into mismatched and ignored when applicable" do
    x = Scientist::Observation.new("x", @experiment) { :x }
    y = Scientist::Observation.new("y", @experiment) { :y }
    z = Scientist::Observation.new("z", @experiment) { :z }

    @experiment.ignore { |control, candidate| candidate == :y }

    result = Scientist::Result.new @experiment, [x, y, z], x

    assert result.mismatched?
    assert result.ignored?
    assert_equal [y], result.ignored
    assert_equal [z], result.mismatched
  end

  it "knows the experiment's name" do
    a = Scientist::Observation.new("a", @experiment) { 1 }
    b = Scientist::Observation.new("b", @experiment) { 1 }
    result = Scientist::Result.new @experiment, [a, b], a

    assert_equal @experiment.name, result.experiment_name
  end

  it "has the context from an experiment" do
    @experiment.context :foo => :bar
    a = Scientist::Observation.new("a", @experiment) { 1 }
    b = Scientist::Observation.new("b", @experiment) { 1 }
    result = Scientist::Result.new @experiment, [a, b], a

    assert_equal({:foo => :bar}, result.context)
  end

end