Merge pull request #5 from UNSW-CEEM/UFLS_upscaling

Upscaling excluding circuits with UFLS dropout

.gitignore

@@ -86,11 +86,11 @@ manufacters_missing_from_cer.csv
 manufacters_missing_from_solar_analytics.csv
 cer_cumulative_capacity_and_number_by_manufacturer_filter_off_grid_formatted.csv
 cer_cumulative_capacity_and_number_by_manufacturer_filter_off_grid.csv=
-manufacters_missing_from_cer.csv
-manufacters_missing_from_solar_analytics.csv
+manufacturers_missing_from_cer.csv
+manufacturers_missing_from_input_db.csv
 cer_cumulative_capacity_and_number_by_manufacturer.csv
 cer_cumulative_capacity_and_number_old.csv
 test_disconnection_summary.csv
-test_manufacters_missing_from_cer.csv
-test_manufacters_missing_from_solar_analytics.csv
+test_manufacturers_missing_from_cer.csv
+test_manufacturers_missing_from_solar_analytics.csv
 test_upscaled_disconnections.csv

shiny.R

@@ -113,7 +113,11 @@ ui <- fluidPage(
           HTML("<br>"),
           uiOutput("save_manufacturer_disconnection_summary"),
           HTML("<br>"),
+          uiOutput("save_manufacturer_disconnection_summary_with_separate_ufls_counts"),
+          HTML("<br>"),
           uiOutput("save_upscaled_disconnection_summary"),
+          HTML("<br>"),
+          uiOutput("save_upscaled_disconnection_summary_with_separate_ufls_counts"),
           HTML("<br><br>"),
           plotlyOutput(outputId="NormPower"),
           plotlyOutput(outputId="Frequency"),
@@ -1022,36 +1026,39 @@ server <- function(input,output,session){
             circuits_to_summarise <- v$circuit_summary
             manufacturer_install_data <- v$manufacturer_install_data
           }
-          disconnection_summary <- group_disconnections_by_manufacturer(circuits_to_summarise)
-          manufacturer_capacitys <- get_manufacturer_capacitys(manufacturer_install_data, load_date(), 
-                                                               region_to_load())
-          disconnection_summary <- join_solar_analytics_and_cer_manufacturer_data(disconnection_summary,
-                                                                                  manufacturer_capacitys)
-          manufacters_missing_from_cer <- get_manufactures_in_solar_analytics_but_not_cer(disconnection_summary)
-          manufacters_missing_from_solar_analytics <- get_manufactures_in_cer_but_not_solar_analytics(disconnection_summary)
-          disconnection_summary <- impose_sample_size_threshold(disconnection_summary, sample_threshold = 30)
-          disconnection_summary <- calc_confidence_intervals_for_disconnections(disconnection_summary)
-          v$disconnection_summary <- calc_upscale_kw_loss(disconnection_summary)
-          v$upscaled_disconnections <- upscale_disconnections(v$disconnection_summary)
-          write.csv(manufacters_missing_from_cer, "logging/manufacters_missing_from_cer.csv", row.names=FALSE)
-          write.csv(manufacters_missing_from_solar_analytics, "logging/manufacters_missing_from_solar_analytics.csv", row.names=FALSE)
+          upscaling_results <- get_upscaling_results(circuits_to_summarise, manufacturer_install_data, load_date(), 
+                                                     region_to_load(), sample_threshold = 30)
+          upscaling_results_with_separate_ufls_counts <- get_upscaling_results_excluding_ufls_affected_circuits(
+            circuits_to_summarise, manufacturer_install_data, load_date(), region_to_load(), sample_threshold = 30)
+
+          v$disconnection_summary <- upscaling_results$disconnection_summary
+          v$upscaled_disconnections <- upscaling_results$upscaled_disconnections
+          v$disconnection_summary_with_separate_ufls_counts <- 
+            upscaling_results_with_separate_ufls_counts$disconnection_summary
+          v$upscaled_disconnections_with_separate_ufls_counts <- 
+            upscaling_results_with_separate_ufls_counts$upscaled_disconnections
 
-          if(length(manufacters_missing_from_cer$manufacturer) > 0) {
-            long_error_message <- c("Some manufacturers present in the solar analytics data could not be ",
+          write.csv(upscaling_results$manufacturers_missing_from_cer, 
+                    "logging/manufacturers_missing_from_cer.csv", row.names=FALSE)
+          write.csv(upscaling_results$manufacturers_missing_from_input_db, 
+                    "logging/manufacturers_missing_from_input_db.csv", row.names=FALSE)
+
+          if(length(upscaling_results$manufacturers_missing_from_cer$manufacturer) > 0) {
+            long_error_message <- c("Some manufacturers present in the input data could not be ",
                                     "matched to the cer data set. A list of these has been saved in the ",
-                                    "file logging/manufacters_missing_from_cer.csv. You may want to review the ", 
-                                    "mapping used in processing the solar analytics data.")
+                                    "file logging/manufacturers_missing_from_cer.csv. You may want to review the ", 
+                                    "mapping used in processing the input data.")
             long_error_message <- paste(long_error_message, collapse = '')
             shinyalert("Manufacturers missing from CER data", long_error_message)
           }
 
-          if(length(manufacters_missing_from_solar_analytics$manufacturer) > 0) {
+          if(length(upscaling_results$manufacturers_missing_from_input_db$manufacturer) > 0) {
             long_error_message <- c("Some manufacturers present in the CER data could not be ",
-                                    "matched to the solar analytics data set. A list of these has been saved in the ",
-                                    "file logging/manufacters_missing_from_solar_analytics.csv. You may wish to review the ", 
+                                    "matched to the input data set. A list of these has been saved in the ",
+                                    "file logging/manufacturers_missing_from_input_db.csv. You may wish to review the ", 
                                     "file to check the number and names of missing manufacturers. ")
             long_error_message <- paste(long_error_message, collapse = '')
-            shinyalert("Manufacturers missing from Solar Analytics data", long_error_message)
+            shinyalert("Manufacturers missing from input data", long_error_message)
           }
 
 
@@ -1100,10 +1107,20 @@ server <- function(input,output,session){
               shinySaveButton("save_manufacturer_disconnection_summary", "Save manufacturer disconnection summary", 
                               "Choose directory for report files ...", filetype=list(xlsx="csv"))
             })
+            output$save_manufacturer_disconnection_summary_with_separate_ufls_counts <- renderUI({
+              shinySaveButton("save_manufacturer_disconnection_summary_with_separate_ufls_counts", 
+                              "Save manufacturer disconnection summary with separate ufls counts", 
+                              "Choose directory for report files ...", filetype=list(xlsx="csv"))
+            })
             output$save_upscaled_disconnection_summary <- renderUI({
               shinySaveButton("save_upscaled_disconnection_summary", "Save upscaled disconnection summary", 
                               "Choose directory for report files ...", filetype=list(xlsx="csv"))
             })
+            output$save_upscaled_disconnection_summary_with_separate_ufls_counts <- renderUI({
+              shinySaveButton("save_upscaled_disconnection_summary_with_separate_ufls_counts", 
+                              "Save upscaled disconnection summary with separate ufls counts", 
+                              "Choose directory for report files ...", filetype=list(xlsx="csv"))
+            })
 
 
             if ("width" %in% names(v$sample_count_table)) {
@@ -1490,6 +1507,18 @@ server <- function(input,output,session){
     }
   })
 
+
+  observeEvent(input$save_manufacturer_disconnection_summary_with_separate_ufls_counts,{
+    volumes <- c(home=getwd())
+    shinyFileSave(input, "save_manufacturer_disconnection_summary_with_separate_ufls_counts", roots=volumes, 
+                  session=session)
+    fileinfo <- parseSavePath(volumes, input$save_manufacturer_disconnection_summary_with_separate_ufls_counts)
+    if (nrow(fileinfo) > 0) {
+      write.csv(v$disconnection_summary_with_separate_ufls_counts, as.character(fileinfo$datapath), row.names=FALSE)
+    }
+  })
+
+
   observeEvent(input$save_upscaled_disconnection_summary,{
     volumes <- c(home=getwd())
     shinyFileSave(input, "save_upscaled_disconnection_summary", roots=volumes, session=session)
@@ -1500,6 +1529,17 @@ server <- function(input,output,session){
   })
 
 
+  observeEvent(input$save_upscaled_disconnection_summary_with_separate_ufls_counts,{
+    volumes <- c(home=getwd())
+    shinyFileSave(input, "save_upscaled_disconnection_summary_with_separate_ufls_counts", roots=volumes, 
+                  session=session)
+    fileinfo <- parseSavePath(volumes, input$save_upscaled_disconnection_summary_with_separate_ufls_counts)
+    if (nrow(fileinfo) > 0) {
+      write.csv(v$upscaled_disconnections_with_separate_ufls_counts, as.character(fileinfo$datapath), row.names=FALSE)
+    }
+  })
+
+
   get_current_settings <- function(){
     settings <- vector(mode='list')
 

upscale_disconnections/example_run_upscaling.R

@@ -0,0 +1,31 @@
+source("load_tool_environment.R")
+
+exclude_solar_edge <- FALSE
+region_to_load <- 'QLD'
+load_start_time <- '2021-05-25'
+circuit_summary <- read.csv(file = "circuit_summary.csv", header = TRUE, stringsAsFactors = FALSE)
+manufacturer_install_data <- read.csv(file = "inbuilt_data/cer_cumulative_capacity_and_number_by_manufacturer.csv", 
+                                      header = TRUE, stringsAsFactors = FALSE)
+
+if (exclude_solar_edge){
+  circuits_to_summarise <- filter(circuit_summary, manufacturer != "SolarEdge")
+  manufacturer_install_data <- filter(manufacturer_install_data, manufacturer != "SolarEdge")
+} else {
+  circuits_to_summarise <- circuit_summary
+  manufacturer_install_data <- manufacturer_install_data
+}
+
+manufacturer_install_data <- calc_installed_capacity_by_standard_and_manufacturer(manufacturer_install_data)
+
+upsc_results <- get_upscaling_results_excluding_ufls_affected_circuits(circuits_to_summarise, manufacturer_install_data,
+                                                                       load_start_time, region_to_load, 30)
+
+disconnection_summary <- upsc_results$disconnection_summary
+upscaled_disconnections <- upsc_results$upscaled_disconnections
+manufacturers_missing_from_cer <- upsc_results$manufacturers_missing_from_cer
+manufacturers_missing_from_input_db <- upsc_results$manufacturers_missing_from_input_db
+
+write.csv(disconnection_summary, "test_disconnection_summary.csv", row.names = FALSE)
+write.csv(upscaled_disconnections, "test_upscaled_disconnections.csv", row.names = FALSE)
+write.csv(manufacturers_missing_from_cer, "test_manufacturers_missing_from_cer.csv", row.names=FALSE)
+write.csv(manufacturers_missing_from_input_db, "test_manufacturers_missing_from_input_db.csv", row.names=FALSE)

upscale_disconnections/post_processing_upscaling.R

@@ -25,16 +25,16 @@ if (exclude_solar_edge){
 disconnection_summary <- group_disconnections_by_manufacturer(circuits_to_summarise)
 manufacturer_capacitys <- get_manufacturer_capacitys(manufacturer_install_data, load_start_time, 
                                                      region_to_load)
-disconnection_summary <- join_solar_analytics_and_cer_manufacturer_data(disconnection_summary,
+disconnection_summary <- join_circuit_summary_and_cer_manufacturer_data(disconnection_summary,
                                                                         manufacturer_capacitys)
-manufacters_missing_from_cer <- get_manufactures_in_solar_analytics_but_not_cer(disconnection_summary)
-manufacters_missing_from_solar_analytics <- get_manufactures_in_cer_but_not_solar_analytics(disconnection_summary)
+manufacturers_missing_from_cer <- get_manufactures_in_input_db_but_not_cer(disconnection_summary)
+manufacturers_missing_from_input_db <- get_manufactures_in_cer_but_not_input_db(disconnection_summary)
 disconnection_summary <- impose_sample_size_threshold(disconnection_summary, sample_threshold = 30)
 disconnection_summary <- calc_confidence_intervals_for_disconnections(disconnection_summary)
 disconnection_summary <- calc_upscale_kw_loss(disconnection_summary)
 upscaled_disconnections <- upscale_disconnections(disconnection_summary)
 
 write.csv(disconnection_summary, "test_disconnection_summary.csv", row.names = FALSE)
 write.csv(upscaled_disconnections, "test_upscaled_disconnections.csv", row.names = FALSE)
-write.csv(manufacters_missing_from_cer, "test_manufacters_missing_from_cer.csv", row.names=FALSE)
-write.csv(manufacters_missing_from_solar_analytics, "test_manufacters_missing_from_solar_analytics.csv", row.names=FALSE)
+write.csv(manufacturers_missing_from_cer, "test_manufacturers_missing_from_cer.csv", row.names=FALSE)
+write.csv(manufacturers_missing_from_input_db, "test_manufacturers_missing_from_input_db.csv", row.names=FALSE)