test

app/evals/test_sentiment_accuracy_eval.rb

@@ -0,0 +1,6 @@
+class TestSentimentAccuracyEval < Leva::BaseEval
+  def evaluate(prediction, expected)
+    score = prediction == expected ? 1.0 : 0.0
+    Leva::Result.new(label: 'sentiment_accuracy', score: score)
+  end
+end

app/models/leva/dataset.rb

@@ -10,7 +10,10 @@
 module Leva
   class Dataset < ApplicationRecord
     has_many :dataset_records, dependent: :destroy
-    has_many :records, through: :dataset_records, source: :recordable
     has_many :experiments, dependent: :destroy
+
+    def add_record(record)
+      dataset_records.create(recordable: record)
+    end
   end
 end

app/models/leva/experiment.rb

@@ -9,7 +9,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  dataset_id :integer          not null
-#  prompt_id  :integer          not null
+#  prompt_id  :integer
 #
 # Indexes
 #

app/runners/test_sentiment_run.rb

@@ -0,0 +1,13 @@
+class TestSentimentRun < Leva::BaseRun
+  def execute(record)
+    # Simple sentiment analysis logic for testing
+    case record.content.downcase
+    when /love|great|excellent/
+      "Positive"
+    when /terrible|bad|awful/
+      "Negative"
+    else
+      "Neutral"
+    end
+  end
+end

db/migrate/20240813173222_create_leva_experiments.rb

@@ -3,7 +3,7 @@ def change
     create_table :leva_experiments do |t|
       t.string :name
       t.references :dataset, null: false, foreign_key: true
-      t.references :prompt, null: false, foreign_key: true
+      t.references :prompt, null: true, foreign_key: true
       t.integer :status
       t.text :metadata
 

test/dummy/db/schema.rb

@@ -42,7 +42,7 @@
   create_table "leva_experiments", force: :cascade do |t|
     t.string "name"
     t.integer "dataset_id", null: false
-    t.integer "prompt_id", null: false
+    t.integer "prompt_id"
     t.integer "status"
     t.text "metadata"
     t.datetime "created_at", null: false

test/models/leva/experiment_test.rb

@@ -9,7 +9,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  dataset_id :integer          not null
-#  prompt_id  :integer          not null
+#  prompt_id  :integer
 #
 # Indexes
 #
@@ -25,8 +25,33 @@
 
 module Leva
   class ExperimentTest < ActiveSupport::TestCase
-    # test "the truth" do
-    #   assert true
-    # end
+    def setup
+      dataset = Leva::Dataset.create(name: "Sentiment Analysis Dataset")
+      dataset.add_record TextContent.create(text: "I love this product!", expected_label: "Positive")
+      dataset.add_record TextContent.create(text: "Terrible experience", expected_label: "Negative")
+      dataset.add_record TextContent.create(text: "I's ok", expected_label: "Neutral")
+      @experiment = Leva::Experiment.create!(name: "Sentiment Analysis", dataset: dataset)
+
+      @run = TestSentimentRun.new
+      @evals = [TestSentimentAccuracyEval.new, TestSentimentF1Eval.new]
+    end
+
+    test "run evaluation with two evals and one runner" do
+      Leva.run_evaluation(experiment: @experiment, run: @run, evals: @evals)
+
+      assert_equal 6, @experiment.evaluation_results.count, "Should have 6 evaluation results (1 run * 3 records * 2 evals)"
+
+      accuracy_results = @experiment.evaluation_results.where(label: 'sentiment_accuracy')
+      f1_results = @experiment.evaluation_results.where(label: 'sentiment_f1')
+
+      assert_equal 3, accuracy_results.count, "Should have 3 accuracy results"
+      assert_equal 3, f1_results.count, "Should have 3 F1 results"
+
+      average_accuracy = accuracy_results.average(:score)
+      average_f1 = f1_results.average(:score)
+
+      assert_in_delta 0.67, average_accuracy, 0.01, "Average accuracy should be about 0.67 (2 out of 3 correct)"
+      assert_in_delta 0.67, average_f1, 0.01, "Average F1 score should be about 0.67 (2 out of 3 correct)"
+    end
   end
 end