Include imputed strata in assessment card/pin

R/recipes.R

@@ -31,7 +31,10 @@ model_main_recipe <- function(data, pred_vars, cat_vars,
     # Remove any variables not an outcome var or in the pred_vars vector
     step_rm(-all_outcomes(), -all_predictors(), -has_role("ID")) %>%
     # Impute missing values using KNN. Specific to condo model, usually used to
-    # impute missing condo building strata
+    # impute missing condo building strata. Within step_impute_knn, an estimated
+    # node value is called with the sample(). This is not deterministic, meaning
+    # different runs of the model will have different imputed values, and thus
+    # different FMVs.
     step_impute_knn(
       all_of(knn_vars),
       neighbors = tune(),

pipeline/02-assess.R

@@ -30,7 +30,7 @@ land_nbhd_rate <- read_parquet(
 
 
 #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# 2. Predict Values ------------------------------------------------------------
+# 2. Predict Values and Recover Strata  ----------------------------------------
 #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 message("Predicting off-market values with trained model")
 
@@ -40,20 +40,61 @@ lgbm_final_full_recipe <- readRDS(paths$output$workflow_recipe$local)
 
 # Load the data for assessment. This is the universe of condo units
 # that need values. Use the trained lightgbm model to estimate a single
-# FMV per unit
+# FMV per unit. Bake the data first so we can extract transformed columns
 assessment_data_pred <- read_parquet(paths$input$assessment$local) %>%
-  as_tibble() %>%
+  as_tibble()
+
+assessment_data_baked <- assessment_data_pred %>%
+  bake(lgbm_final_full_recipe, new_data = ., all_predictors())
+
+assessment_data_pred <- assessment_data_pred %>%
   mutate(
-    pred_card_initial_fmv = predict(
+    .,
+    pred_card_initial_fmv = as.numeric(predict(
       lgbm_final_full_fit,
-      new_data = bake(
-        lgbm_final_full_recipe,
-        new_data = .,
-        all_predictors()
-      )
-    )$.pred
+      new_data = assessment_data_baked
+    )$.pred),
+    # Strata variables are converted to 0-indexed integers during baking.
+    # We save those converted values so we can unconvert them below
+    temp_strata_1 = assessment_data_baked$meta_strata_1,
+    temp_strata_2 = assessment_data_baked$meta_strata_2
   )
 
+# The baked data encodes categorical values as base-0 integers.
+# However, here we want to recover the original (unencoded) values of our
+# strata variables wherever they've been imputed by the baking step. To do so,
+# we create a mapping of the encoded to unencoded values and use them to
+# recover both the original strata values and those imputed by
+# step_impute_knn (in R/recipes.R)
+strata_mapping_1 <- assessment_data_pred %>%
+  filter(!is.na(meta_strata_1)) %>%
+  distinct(temp_strata_1, meta_strata_1) %>%
+  pull(meta_strata_1, name = temp_strata_1)
+strata_mapping_2 <- assessment_data_pred %>%
+  filter(!is.na(meta_strata_2)) %>%
+  distinct(temp_strata_2, meta_strata_2) %>%
+  pull(meta_strata_2, name = temp_strata_2)
+
+# Recover the imputed strata values
+assessment_data_pred <- assessment_data_pred %>%
+  mutate(
+    # Binary variable to identify condos which have imputed strata
+    flag_strata_is_imputed = is.na(meta_strata_1) | is.na(meta_strata_2),
+    # Use mappings to replace meta_strata_1 and meta_strata_2 directly
+    meta_strata_1 = ifelse(
+      is.na(meta_strata_1),
+      unname(strata_mapping_1[as.character(temp_strata_1)]),
+      meta_strata_1
+    ),
+    meta_strata_2 = ifelse(
+      is.na(meta_strata_2),
+      unname(strata_mapping_2[as.character(temp_strata_2)]),
+      meta_strata_2
+    )
+  ) %>%
+  # Remove unnecessary columns
+  select(-temp_strata_1, -temp_strata_2)
+
 
 
 
@@ -154,14 +195,15 @@ assessment_data_merged %>%
   select(
     meta_year, meta_pin, meta_class, meta_card_num, meta_lline_num,
     meta_modeling_group, ends_with("_num_sale"), pred_card_initial_fmv,
-    all_of(params$model$predictor$all), township_code
+    all_of(params$model$predictor$all),
+    flag_strata_is_imputed, township_code
   ) %>%
   mutate(
     ccao_n_years_exe_homeowner = as.integer(ccao_n_years_exe_homeowner)
   ) %>%
   ccao::vars_recode(
-    starts_with("char_"),
-    type = "long",
+    cols = starts_with("char_"),
+    code_type = "long",
     as_factor = FALSE
   ) %>%
   write_parquet(paths$output$assessment_card$local)
@@ -203,7 +245,7 @@ sales_data_two_most_recent <- sales_data %>%
     meta_pin, meta_year,
     meta_sale_price, meta_sale_date, meta_sale_document_num,
     sv_outlier_reason1, sv_outlier_reason2, sv_outlier_reason3,
-    meta_sale_num_parcels, sv_added_later
+    meta_sale_num_parcels
   ) %>%
   # Include outliers, since these data are used for desk review and
   # not for modeling
@@ -225,8 +267,7 @@ sales_data_two_most_recent <- sales_data %>%
       meta_sale_outlier_reason1,
       meta_sale_outlier_reason2,
       meta_sale_outlier_reason3,
-      meta_sale_num_parcels,
-      sv_added_later
+      meta_sale_num_parcels
     ),
     names_glue = "{mr}_{gsub('meta_sale_', '', .value)}"
   ) %>%
@@ -270,7 +311,8 @@ assessment_data_pin <- assessment_data_merged %>%
     meta_year, meta_pin, meta_pin10, meta_triad_code, meta_township_code,
     meta_nbhd_code, meta_tax_code, meta_class, meta_tieback_key_pin,
     meta_tieback_proration_rate, meta_cdu, meta_modeling_group,
-    meta_pin_num_landlines, char_yrblt,
+    meta_pin_num_landlines, meta_strata_1, meta_strata_2,
+    flag_strata_is_imputed, char_yrblt,
 
     # Keep overall building square footage
     char_total_bldg_sf = char_building_sf,
@@ -389,7 +431,7 @@ message("Saving final PIN-level data")
 assessment_data_pin_final %>%
   ccao::vars_recode(
     cols = starts_with("char_"),
-    type = "short",
+    code_type = "short",
     as_factor = FALSE
   ) %>%
   select(-meta_pin10) %>%