From 0c0e7ef70993d3b7df272e4895fe8a8c71104406 Mon Sep 17 00:00:00 2001 From: dcooley Date: Tue, 4 Feb 2020 07:33:30 -0800 Subject: [PATCH] debugging #58 --- R/scratch2.R | 8 +-- inst/include/jsonify/from_json/api.hpp | 6 +- inst/include/jsonify/from_json/from_json.hpp | 15 ++-- .../jsonify/from_json/from_json_utils.hpp | 36 +++++----- .../jsonify/from_json/simplify/simplify.hpp | 54 ++++++++------ inst/include/jsonify/to_json/dates/dates.hpp | 20 +++--- .../jsonify/to_json/writers/complex.hpp | 12 ++-- .../jsonify/to_json/writers/simple.hpp | 70 ++++++++++--------- src/tests.cpp | 2 +- src/validate.cpp | 5 +- 10 files changed, 122 insertions(+), 106 deletions(-) diff --git a/R/scratch2.R b/R/scratch2.R index 4285f07..7eb66e2 100644 --- a/R/scratch2.R +++ b/R/scratch2.R @@ -3,7 +3,7 @@ # # json <- "~/Desktop/google_timeline/Takeout/Location History/Location History.json" # -# js <- jsonlite::fromJSON(json) +# # js <- jsonlite::fromJSON(json) # # # # js2 <- jsonify::from_json(json) # @@ -11,11 +11,11 @@ # jfy <- from_json( json_small, simplify = T, fill_na = T) # jlt <- fromJSON( json_small ) # -# ls <- readLines(json) # -# js <- paste0(ls, collapse = "") # -# #jsonify::validate_json( js ) +# +# json <- "~/Desktop/google_timeline/Takeout/Location History/Location History Reducing.json" +# ls <- readLines(json); js <- paste0(ls, collapse = "") # # res <- jsonify::from_json( js, fill_na = T ) # diff --git a/inst/include/jsonify/from_json/api.hpp b/inst/include/jsonify/from_json/api.hpp index 64daf4e..88555f0 100644 --- a/inst/include/jsonify/from_json/api.hpp +++ b/inst/include/jsonify/from_json/api.hpp @@ -16,7 +16,7 @@ namespace api { //' @export inline SEXP from_json(rapidjson::Value& doc, bool& simplify, bool& fill_na ) { - int sequential_array_counter = 0; + R_xlen_t sequential_array_counter = 0; // If the input is a scalar value of type int, double, string, or bool, // return Rcpp vector with length 1. @@ -44,9 +44,7 @@ namespace api { return x; } - R_xlen_t depth = 0; - - return jsonify::from_json::json_to_sexp( doc, simplify, fill_na, sequential_array_counter, depth ); + return jsonify::from_json::json_to_sexp( doc, simplify, fill_na, sequential_array_counter ); } inline SEXP from_json( const char* json, bool& simplify, bool& fill_na ) { diff --git a/inst/include/jsonify/from_json/from_json.hpp b/inst/include/jsonify/from_json/from_json.hpp index 23c853d..7adc7c5 100644 --- a/inst/include/jsonify/from_json/from_json.hpp +++ b/inst/include/jsonify/from_json/from_json.hpp @@ -13,15 +13,14 @@ namespace from_json { const rapidjson::Value& json, bool& simplify, bool& fill_na, - int sequential_array_counter, - R_xlen_t& depth + R_xlen_t sequential_array_counter ) { Rcpp::List res(1); int json_type = json.GetType(); - int json_length = json.Size(); + R_xlen_t json_length = json.Size(); if(json_length == 0) { if( json_type == 4 ) { @@ -34,8 +33,6 @@ namespace from_json { R_xlen_t i; - - depth = depth + 1; //Rcpp::Rcout << "depth: " << depth << std::endl; //Rcpp::Rcout << "i: " << i << std::endl; @@ -91,11 +88,11 @@ namespace from_json { // array case rapidjson::kArrayType: { const rapidjson::Value& temp_array = itr->value; - out[i] = json_to_sexp( temp_array, simplify, fill_na, sequential_array_counter, depth ); + out[i] = json_to_sexp( temp_array, simplify, fill_na, sequential_array_counter ); break; } case rapidjson::kObjectType: { - out[i] = json_to_sexp( itr->value, simplify, fill_na, sequential_array_counter, depth ); + out[i] = json_to_sexp( itr->value, simplify, fill_na, sequential_array_counter ); break; } @@ -164,7 +161,7 @@ namespace from_json { } // array case rapidjson::kArrayType: { - array_of_array[i] = json_to_sexp( json[i], simplify, fill_na, sequential_array_counter, depth ); + array_of_array[i] = json_to_sexp( json[i], simplify, fill_na, sequential_array_counter ); sequential_array_counter++; break; } @@ -172,7 +169,7 @@ namespace from_json { case rapidjson::kObjectType: { sequential_array_counter = 0; const rapidjson::Value& temp_val = json[i]; - array_of_array[i] = json_to_sexp( temp_val, simplify, fill_na, sequential_array_counter, depth ); + array_of_array[i] = json_to_sexp( temp_val, simplify, fill_na, sequential_array_counter ); break; } default: { diff --git a/inst/include/jsonify/from_json/from_json_utils.hpp b/inst/include/jsonify/from_json/from_json_utils.hpp index 1e26675..b144774 100644 --- a/inst/include/jsonify/from_json/from_json_utils.hpp +++ b/inst/include/jsonify/from_json/from_json_utils.hpp @@ -11,11 +11,12 @@ namespace jsonify { namespace from_json { - inline int where_is( - Rcpp::String to_find, - Rcpp::StringVector& sv ) { - int n = sv.size(); - int i; + inline R_xlen_t where_is( + Rcpp::String to_find, + Rcpp::StringVector& sv + ) { + R_xlen_t n = sv.size(); + R_xlen_t i; for( i = 0; i < n; i++ ) { if ( to_find == sv[i] ) { return i; @@ -32,10 +33,11 @@ namespace from_json { std::unordered_set< int > dtypes; - int doc_len = doc.Size(); + R_xlen_t doc_len = doc.Size(); int curr_dtype; - for(int i = 0; i < doc_len; ++i) { + R_xlen_t i; + for(i = 0; i < doc_len; ++i) { curr_dtype = doc[i].GetType(); // rapidjson uses separate ints for types true (2) and false (1)...combine // them into one value such that bool is 1. @@ -105,18 +107,18 @@ namespace from_json { return 0; } - // Convert all NULL elements in a list to NA. - inline void null_to_na(Rcpp::List& x) { - for(unsigned int i = 0; i < x.size(); ++i) { - if(Rf_isNull(x[i])) { - x[i] = R_NA_VAL; - } - } - } + // // Convert all NULL elements in a list to NA. + // inline void null_to_na(Rcpp::List& x) { + // for(unsigned int i = 0; i < x.size(); ++i) { + // if(Rf_isNull(x[i])) { + // x[i] = R_NA_VAL; + // } + // } + // } // returns -1 if doens't exist // else the stored r_type - inline int column_value( + inline R_xlen_t column_value( std::unordered_map< std::string, int >& column_map, const char* to_find ) { @@ -125,7 +127,7 @@ namespace from_json { it = column_map.find( str ); if( it != column_map.end() ) { - int res = it->second; + R_xlen_t res = it->second; return res; } return -1; diff --git a/inst/include/jsonify/from_json/simplify/simplify.hpp b/inst/include/jsonify/from_json/simplify/simplify.hpp index 30f9f98..88451c4 100644 --- a/inst/include/jsonify/from_json/simplify/simplify.hpp +++ b/inst/include/jsonify/from_json/simplify/simplify.hpp @@ -149,8 +149,8 @@ namespace from_json { inline SEXP simplify_matrix( Rcpp::List& out, - int& n_col, - int& n_row, + R_xlen_t& n_col, + R_xlen_t& n_row, int& r_type ) { @@ -168,10 +168,13 @@ namespace from_json { // } else { Rcpp::IntegerMatrix mat( n_row, n_col ); - for( int i = 0; i < n_row; i++ ) { + R_xlen_t i; + R_xlen_t j; + + for( i = 0; i < n_row; i++ ) { Rcpp::IntegerVector this_vec = out[i]; - for( int j = 0; j < n_col; j++ ) { - int this_val = this_vec[j]; + for( j = 0; j < n_col; j++ ) { + R_xlen_t this_val = this_vec[j]; mat( i, j ) = this_val; } } @@ -190,9 +193,12 @@ namespace from_json { // } else { Rcpp::NumericMatrix mat( n_row, n_col ); - for( int i = 0; i < n_row; i++ ) { + R_xlen_t i; + R_xlen_t j; + + for( i = 0; i < n_row; i++ ) { Rcpp::NumericVector this_vec = out[i]; - for( int j = 0; j < n_col; j++ ) { + for( j = 0; j < n_col; j++ ) { double this_val = this_vec[j]; mat( i, j ) = this_val; } @@ -212,9 +218,12 @@ namespace from_json { // } else { Rcpp::LogicalMatrix mat( n_row, n_col ); - for( int i = 0; i < n_row; i++ ) { + R_xlen_t i; + R_xlen_t j; + + for( i = 0; i < n_row; i++ ) { Rcpp::LogicalVector this_vec = out[i]; - for( int j = 0; j < n_col; j++ ) { + for( j = 0; j < n_col; j++ ) { bool this_val = this_vec[j]; mat( i, j ) = this_val; } @@ -238,9 +247,12 @@ namespace from_json { // return mat; // } else { Rcpp::StringMatrix mat( n_row, n_col ); - for( int i = 0; i < n_row; i++ ) { + R_xlen_t i; + R_xlen_t j; + + for( i = 0; i < n_row; i++ ) { Rcpp::StringVector this_vec = out[i]; - for( int j = 0; j < n_col; j++ ) { + for( j = 0; j < n_col; j++ ) { Rcpp::String this_val = this_vec[j]; mat( i, j ) = this_val; } @@ -260,6 +272,7 @@ namespace from_json { std::unordered_set array_lengths; std::unordered_set array_types; bool can_be_matrix = true; + for( j = 0; j < n; j++ ) { SEXP s = array_of_array[j]; int this_type = TYPEOF( s ); @@ -281,8 +294,9 @@ namespace from_json { if( can_be_matrix ) { Rcpp::IntegerVector arr_types( array_types.begin(), array_types.end() ); int r_type = Rcpp::max( arr_types ); - int n_col = *array_lengths.begin(); // only one sizez - int n_row = n; + R_xlen_t n_col = *array_lengths.begin(); // only one sizez + R_xlen_t n_row = n; + return jsonify::from_json::simplify_matrix( array_of_array, n_col, n_row, r_type ); } else { return array_of_array; @@ -437,7 +451,7 @@ namespace from_json { inline SEXP simplify_dataframe_fill_na( Rcpp::List& out, - int& doc_len + R_xlen_t& doc_len ) { // the number of rows is equal to the number of list elements? @@ -454,10 +468,10 @@ namespace from_json { std::unordered_map< std::string, int > column_lengths; int struct_type; - int sexp_length; + R_xlen_t sexp_length; int tp; int st; - int ln; + R_xlen_t ln; Rcpp::StringVector list_names; std::vector< std::string > column_names; @@ -569,7 +583,7 @@ namespace from_json { // iff any column lenghts are different, it's a list inline SEXP simplify_dataframe( Rcpp::List& out, - int& doc_len + R_xlen_t& doc_len ) { // the number of rows is equal to the number of list elements? @@ -586,10 +600,10 @@ namespace from_json { std::unordered_map< std::string, int > column_lengths; int struct_type; - int sexp_length; + R_xlen_t sexp_length; int tp; int st; - int ln; + R_xlen_t ln; Rcpp::StringVector list_names; @@ -611,7 +625,7 @@ namespace from_json { for( j = 0; j < list_size; j++ ) { const char* this_name = list_names[j]; Rcpp::StringVector these_names = this_list.names(); - int found_name = where_is( this_name, these_names ); + R_xlen_t found_name = where_is( this_name, these_names ); if( found_name == -1 ) { // can't simplify diff --git a/inst/include/jsonify/to_json/dates/dates.hpp b/inst/include/jsonify/to_json/dates/dates.hpp index 4e775a9..350df46 100644 --- a/inst/include/jsonify/to_json/dates/dates.hpp +++ b/inst/include/jsonify/to_json/dates/dates.hpp @@ -10,8 +10,8 @@ namespace jsonify { namespace dates { inline bool is_in( const char* x, Rcpp::CharacterVector v ) { - int n = v.size(); - int i; + R_xlen_t n = v.size(); + R_xlen_t i; for( i = 0; i < n; i++ ) { if( v[i] == x ) { return true; @@ -22,8 +22,8 @@ namespace dates { inline Rcpp::StringVector date_to_string( Rcpp::IntegerVector& iv ) { - int i; - int n = iv.size(); + R_xlen_t i; + R_xlen_t n = iv.size(); Rcpp::StringVector sv( n ); for ( i = 0; i < n; i++ ) { @@ -37,8 +37,8 @@ namespace dates { inline Rcpp::StringVector date_to_string( Rcpp::NumericVector& nv ) { - int i; - int n = nv.size(); + R_xlen_t i; + R_xlen_t n = nv.size(); Rcpp::StringVector sv( n ); for ( i = 0; i < n; i++ ) { @@ -52,8 +52,8 @@ namespace dates { inline Rcpp::StringVector posixct_to_string( Rcpp::IntegerVector& iv ) { - int i; - int n = iv.size(); + R_xlen_t i; + R_xlen_t n = iv.size(); Rcpp::StringVector sv( n ); @@ -75,8 +75,8 @@ namespace dates { inline Rcpp::StringVector posixct_to_string( Rcpp::NumericVector& nv ) { - int i; - int n = nv.size(); + R_xlen_t i; + R_xlen_t n = nv.size(); boost::local_time::tz_database tz_db; diff --git a/inst/include/jsonify/to_json/writers/complex.hpp b/inst/include/jsonify/to_json/writers/complex.hpp index 632065f..8b90bc9 100644 --- a/inst/include/jsonify/to_json/writers/complex.hpp +++ b/inst/include/jsonify/to_json/writers/complex.hpp @@ -76,7 +76,7 @@ namespace complex { int digits, bool numeric_dates, bool factors_as_string, - int row + R_xlen_t row ) { switch( TYPEOF( this_vec ) ) { @@ -136,11 +136,11 @@ namespace complex { bool numeric_dates = true, bool factors_as_string = true, std::string by = "row", - int row = -1, // for when we are recursing into a row of a data.frame + R_xlen_t row = -1, // for when we are recursing into a row of a data.frame bool in_data_frame = false // for keeping track of when we're in a column of a data.frame ) { - int i, df_col, df_row; + R_xlen_t i, df_col, df_row; if( Rf_isNull( list_element ) ) { writer.StartObject(); @@ -176,8 +176,8 @@ namespace complex { in_data_frame = true; Rcpp::DataFrame df = Rcpp::as< Rcpp::DataFrame >( list_element ); - int n_cols = df.ncol(); - int n_rows = df.nrows(); + R_xlen_t n_cols = df.ncol(); + R_xlen_t n_rows = df.nrows(); Rcpp::StringVector column_names = df.names(); // issue 59 @@ -313,7 +313,7 @@ namespace complex { } else { lst = temp_lst; - int n = lst.size(); + R_xlen_t n = lst.size(); if ( n == 0 ) { writer.StartArray(); diff --git a/inst/include/jsonify/to_json/writers/simple.hpp b/inst/include/jsonify/to_json/writers/simple.hpp index 120529f..98ea8dc 100644 --- a/inst/include/jsonify/to_json/writers/simple.hpp +++ b/inst/include/jsonify/to_json/writers/simple.hpp @@ -22,11 +22,12 @@ namespace simple { bool unbox ) { - int n = sv.size(); + R_xlen_t n = sv.size(); bool will_unbox = jsonify::utils::should_unbox( n, unbox ); jsonify::utils::start_array( writer, will_unbox ); + R_xlen_t i; - for ( int i = 0; i < n; i++ ) { + for ( i = 0; i < n; i++ ) { if (Rcpp::StringVector::is_na( sv[i] ) ) { writer.Null(); } else{ @@ -43,7 +44,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::StringVector& sv, - int row + R_xlen_t row ) { if ( Rcpp::StringVector::is_na( sv[ row ] ) ) { @@ -77,12 +78,13 @@ namespace simple { } else { - int n = nv.size(); + R_xlen_t n = nv.size(); bool will_unbox = jsonify::utils::should_unbox( n, unbox ); jsonify::utils::start_array( writer, will_unbox ); + R_xlen_t i; - for ( int i = 0; i < n; i++ ) { + for ( i = 0; i < n; i++ ) { if( Rcpp::NumericVector::is_na( nv[i] ) ) { writer.Null(); } else { @@ -100,7 +102,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::NumericVector& nv, - int row, + R_xlen_t row, int digits, bool numeric_dates ) { @@ -158,7 +160,7 @@ namespace simple { // no levels - from NA_character_ vector Rcpp::StringVector s(1); s[0] = NA_STRING; - int ele = 0; + R_xlen_t ele = 0; write_value( writer, s, ele ); } else { Rcpp::StringVector str = Rcpp::as< Rcpp::StringVector >( iv ); @@ -168,11 +170,12 @@ namespace simple { } else { - int n = iv.size(); + R_xlen_t n = iv.size(); bool will_unbox = jsonify::utils::should_unbox( n, unbox ); jsonify::utils::start_array( writer, will_unbox ); + R_xlen_t i; - for ( int i = 0; i < n; i++ ) { + for ( i = 0; i < n; i++ ) { if( Rcpp::IntegerVector::is_na( iv[i] ) ) { writer.Null(); } else { @@ -191,14 +194,13 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::IntegerVector& iv, - int row, + R_xlen_t row, bool numeric_dates, bool factors_as_string ) { Rcpp::CharacterVector cls = jsonify::utils::getRClass( iv ); - if( !numeric_dates && jsonify::dates::is_in( "Date", cls ) ) { Rcpp::StringVector sv = jsonify::dates::date_to_string( iv ); @@ -219,7 +221,7 @@ namespace simple { // no level s- from NA_character_ vector Rcpp::StringVector s(1); s[0] = NA_STRING; - int ele = 0; + R_xlen_t ele = 0; write_value( writer, s, ele ); } else { Rcpp::StringVector str = Rcpp::as< Rcpp::StringVector >( iv ); @@ -244,11 +246,12 @@ namespace simple { bool unbox, int digits ) { - int n = nv.size(); + R_xlen_t n = nv.size(); bool will_unbox = jsonify::utils::should_unbox( n, unbox ); jsonify::utils::start_array( writer, will_unbox ); + R_xlen_t i; - for ( int i = 0; i < n; i++ ) { + for ( i = 0; i < n; i++ ) { if (Rcpp::NumericVector::is_na( nv[i] ) ) { writer.Null(); } else { @@ -265,11 +268,12 @@ namespace simple { Rcpp::IntegerVector& iv, bool unbox ) { - int n = iv.size(); + R_xlen_t n = iv.size(); bool will_unbox = jsonify::utils::should_unbox( n, unbox ); jsonify::utils::start_array( writer, will_unbox ); + R_xlen_t i; - for ( int i = 0; i < n; i++ ) { + for ( i = 0; i < n; i++ ) { if (Rcpp::IntegerVector::is_na( iv[i] ) ) { writer.Null(); } else { @@ -287,11 +291,11 @@ namespace simple { bool unbox ) { - int n = lv.size(); + R_xlen_t n = lv.size(); bool will_unbox = jsonify::utils::should_unbox( n, unbox ); jsonify::utils::start_array( writer, will_unbox ); - - for ( int i = 0; i < n; i++ ) { + R_xlen_t i; + for ( i = 0; i < n; i++ ) { if (Rcpp::LogicalVector::is_na( lv[i] ) ) { writer.Null(); } else { @@ -306,7 +310,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::LogicalVector& lv, - int row + R_xlen_t row ) { if ( Rcpp::LogicalVector::is_na( lv[ row ] ) ) { writer.Null(); @@ -362,7 +366,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::IntegerMatrix& mat, - int& row, + R_xlen_t& row, bool unbox = false ) { @@ -380,8 +384,8 @@ namespace simple { bool will_unbox = false; jsonify::utils::start_array( writer, will_unbox ); - int n; - int i; + R_xlen_t n; + R_xlen_t i; if ( by == "row" ) { n = mat.nrow(); @@ -403,7 +407,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::NumericMatrix& mat, - int& row, + R_xlen_t& row, bool unbox = false ) { @@ -424,8 +428,8 @@ namespace simple { bool will_unbox = false; jsonify::utils::start_array( writer, will_unbox ); - int n; - int i; + R_xlen_t n; + R_xlen_t i; if ( by == "row" ) { n = mat.nrow(); for ( i = 0; i < n; i++ ) { @@ -446,7 +450,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::StringMatrix& mat, - int& row, + R_xlen_t& row, bool unbox = false ) { @@ -464,8 +468,8 @@ namespace simple { bool will_unbox = false; jsonify::utils::start_array( writer, will_unbox ); - int i; - int n; + R_xlen_t i; + R_xlen_t n; if( by == "row" ) { n = mat.nrow(); @@ -487,7 +491,7 @@ namespace simple { inline void write_value( Writer& writer, Rcpp::LogicalMatrix& mat, - int& row, + R_xlen_t& row, bool unbox = false ) { @@ -505,8 +509,8 @@ namespace simple { bool will_unbox = false; jsonify::utils::start_array( writer, will_unbox ); - int i; - int n; + R_xlen_t i; + R_xlen_t n; if( by == "row" ) { n = mat.nrow(); @@ -534,7 +538,7 @@ namespace simple { inline void write_value( Writer& writer, SEXP sexp, - int row, + R_xlen_t row, int digits, bool numeric_dates, bool factors_as_string diff --git a/src/tests.cpp b/src/tests.cpp index 5b9c508..088276f 100644 --- a/src/tests.cpp +++ b/src/tests.cpp @@ -36,7 +36,7 @@ void source_tests() { Rcpp::LogicalVector lv; bool unbox, numeric_dates, factors_as_string; int digits; - int row; + R_xlen_t row; sv = Rcpp::StringVector::create("a"); unbox = false; diff --git a/src/validate.cpp b/src/validate.cpp index 6aa5378..9ccb865 100644 --- a/src/validate.cpp +++ b/src/validate.cpp @@ -4,9 +4,10 @@ // [[Rcpp::export]] Rcpp::LogicalVector rcpp_validate_json( Rcpp::StringVector json ) { - int n = json.size(); + R_xlen_t n = json.size(); + R_xlen_t i; Rcpp::LogicalVector res( n ); - for ( int i = 0; i < n; i++ ) { + for ( i = 0; i < n; i++ ) { res[i] = jsonify::validate::validate_json( json[i] ); } return res;