Back to Repositories

Validating Experiment Control Flow in GitHub Scientist

This test suite validates the core functionality of the Scientist library, focusing on experiment execution, context handling, and behavior selection. It ensures reliable operation of the scientific method pattern for feature comparisons and rollouts.

Test Coverage Overview

The test suite provides comprehensive coverage of Scientist’s key features:

Version verification
Experiment instantiation and execution
Context management
Behavior selection logic
Module-level and object-level experiment handling

Tests validate both success paths and edge cases like nil test names and context inheritance.

Implementation Analysis

The testing approach uses RSpec’s describe/it pattern for behavioral specifications.

Modular test organization by feature
Isolation of behaviors using Object.new instances
Explicit verification of return values and side effects
Clear separation between control and candidate code paths

Technical Details

Testing infrastructure includes:

RSpec as the testing framework
Ruby assertions for validations
Dynamic object extension with Scientist module
Behavior tracking through array collection
Context manipulation via hash structures

Best Practices Demonstrated

The test suite exemplifies several testing best practices:

Isolated test cases with clear assertions
Comprehensive feature coverage
Explicit context management
Clean setup and teardown
Clear behavior verification

github/scientist

test/scientist_test.rb

            
describe Scientist do
  it "has a version or whatever" do
    assert Scientist::VERSION
  end

  it "provides a helper to instantiate and run experiments" do
    obj = Object.new
    obj.extend(Scientist)

    r = obj.science "test" do |e|
      e.use { :control }
      e.try { :candidate }
    end

    assert_equal :control, r
  end

  it "provides a module method to instantiate and run experiments" do
    r = Scientist.run "test" do |e|
      e.use { :control }
      e.try { :candidate }
    end

    assert_equal :control, r
  end

  it "provides an empty default_scientist_context" do
    obj = Object.new
    obj.extend(Scientist)

    assert_equal Hash.new, obj.default_scientist_context
  end

  it "respects default_scientist_context" do
    obj = Object.new
    obj.extend(Scientist)

    def obj.default_scientist_context
      { :default => true }
    end

    experiment = nil

    obj.science "test" do |e|
      experiment = e
      e.context :inline => true
      e.use { }
    end

    refute_nil experiment
    assert_equal true, experiment.context[:default]
    assert_equal true, experiment.context[:inline]
  end

  it "runs the named test instead of the control" do
    obj = Object.new
    obj.extend(Scientist)

    behaviors_executed = []

    result = obj.science "test", run: "first-way" do |e|
      e.try("first-way") { behaviors_executed << "first-way" ; true }
      e.try("second-way") { behaviors_executed << "second-way" ; true }
    end

    assert_equal true, result
    assert_equal [ "first-way" ], behaviors_executed
  end

  it "runs control when there is a nil named test" do
    obj = Object.new
    obj.extend(Scientist)

    behaviors_executed = []

    result = obj.science "test", nil do |e|
      e.use { behaviors_executed << "control" ; true }
      e.try("second-way") { behaviors_executed << "second-way" ; true }
    end

    assert_equal true, result
    assert_equal [ "control" ], behaviors_executed
  end
end