From d2cd4655a5a765ec4ce6441f7057f71eb76c0139 Mon Sep 17 00:00:00 2001
From: Ted Middleton <middleton.ted@gmail.com>
Date: Tue, 3 May 2022 08:25:26 -0700
Subject: [PATCH] Enhancement for frame::new_series()

frame::new_series() should do the right thing when the new column is
missing disallowed but the expression returns a missing-allowed value
(mi<T>). That means calling default ctor's for missing values.

TODO

- Can we change binary_expr et. al. to use operator() instead of
  get_value()? That way frame::new_series(), frame::rows(), and any
  other function that we craft to take expression objects could also
  take lambdas as well.

- Correlations

- Add a function call wrapper for function calls in expressions

auto f2 = f1.new_column<double>( "ceil", _f( std::ceil, _0 ) );

- csv parser
---
 mainframe/base.hpp            | 25 +++++++++++++++++++++++++
 mainframe/frame.hpp           | 15 ++++++++++++---
 tests/mainframe_test_main.cpp | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/mainframe/base.hpp b/mainframe/base.hpp
index 874b1e5..11bbd54 100644
--- a/mainframe/base.hpp
+++ b/mainframe/base.hpp
@@ -100,6 +100,31 @@ struct ensure_missing< mi<T> >
     using type = mi<T>;
 };
 
+template< typename T >
+struct unwrap_missing
+{
+    using type = T;
+    static T unwrap( const T& t ) 
+    {
+        return t;
+    }
+};
+
+template< typename T >
+struct unwrap_missing< mi<T> >
+{
+    using type = T;
+    static T unwrap( const mi<T>& t ) 
+    {
+        if ( t.has_value() ) {
+            return *t;
+        }
+        else {
+            // default construction - our last resort
+            return T{};
+        }
+    }
+};
 
 } // namespace mf
 
diff --git a/mainframe/frame.hpp b/mainframe/frame.hpp
index 3793fef..7a6c676 100644
--- a/mainframe/frame.hpp
+++ b/mainframe/frame.hpp
@@ -537,9 +537,18 @@ class frame
         auto b = out.begin();
         auto e = out.end();
         auto it = b;
-        for ( ; it != e; ++it ) {
-            auto val = expr.get_value( b, it, e );
-            it->template at<sizeof...(Ts)>() = val;
+        if constexpr ( is_missing<T>::value ) {
+            for ( ; it != e; ++it ) {
+                auto val = expr.get_value( b, it, e );
+                it->template at<sizeof...(Ts)>() = val;
+            }
+        } 
+        else {
+            for ( ; it != e; ++it ) {
+                auto val = expr.get_value( b, it, e );
+                auto uval = unwrap_missing<decltype(val)>::unwrap(val);
+                it->template at<sizeof...(Ts)>() = uval;
+            }
         }
         return out;
     }
diff --git a/tests/mainframe_test_main.cpp b/tests/mainframe_test_main.cpp
index d7efa90..b7681c1 100644
--- a/tests/mainframe_test_main.cpp
+++ b/tests/mainframe_test_main.cpp
@@ -1022,6 +1022,39 @@ TEST_CASE( "expression offsets", "[frame]" )
         REQUIRE( (b + 7)->at( _3 ) == 2021_y/January/9 );
         REQUIRE( (b + 8)->at( _3 ) == missing );
     }
+
+    SECTION( "initially disallow missing into disallow missing" )
+    {
+        frame<year_month_day, double, bool> f1;
+        f1.set_column_names( "date", "temperature", "rain" );
+        f1.push_back( 2022_y/January/1, 8.9, false );
+        f1.push_back( 2022_y/January/2, 10.0, false );
+        f1.push_back( 2022_y/January/3, 11.1, true );
+        f1.push_back( 2022_y/January/4, 12.2, false );
+        f1.push_back( 2022_y/January/5, 13.3, false );
+        f1.push_back( 2022_y/January/6, 14.4, true );
+        f1.push_back( 2022_y/January/7, 15.5, false );
+        f1.push_back( 2022_y/January/8, 9.1, true );
+        f1.push_back( 2022_y/January/9, 9.3, false );
+
+        auto f2 = f1.new_series<year_month_day>( "yesterday", _0[-1] );
+        dout << f2;
+        REQUIRE( f2.num_columns() == 4 );
+        auto b = f2.begin();
+        REQUIRE( (b + 0)->at( _3 ) == year_month_day{} );
+        REQUIRE( (b + 1)->at( _3 ) == 2022_y/January/1 );
+        REQUIRE( (b + 2)->at( _3 ) == 2022_y/January/2 );
+        REQUIRE( (b + 8)->at( _3 ) == 2022_y/January/8 );
+
+        auto f3 = f1.new_series<year_month_day>( "tomorrow last year", _0[+1] - years(1) );
+        dout << f3;
+        REQUIRE( f3.num_columns() == 4 );
+        b = f3.begin();
+        REQUIRE( (b + 0)->at( _3 ) == 2021_y/January/2 );
+        REQUIRE( (b + 1)->at( _3 ) == 2021_y/January/3 );
+        REQUIRE( (b + 7)->at( _3 ) == 2021_y/January/9 );
+        REQUIRE( (b + 8)->at( _3 ) == year_month_day{} );
+    }
 }
 
 TEST_CASE( "drop_missing()", "[frame]" )