---
title: "Get CPA Sample"
output:
  html_notebook:
    highlight: tango
    fig_caption: yes
    theme: journal
    toc: yes
    toc_float:
      collapsed: yes
      smooth_scroll: yes
---

```{r setup, include=FALSE, purl=FALSE}
knitr::opts_knit$set(root.dir = normalizePath(here::here()))
knitr::opts_chunk$set(root.dir = normalizePath(here::here()))
```
 
# Utils
```{r}
cache_obj <- function(.expr = { }, .name, .dir, .rerun = FALSE) {
  # Create Directory -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- ---
  dir_ <- fs::dir_create(.dir)
  fil_ <- file.path(dir_, paste0(.name, ".fst"))

  # Check if exists -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -
  exist_ <- file.exists(fil_)

  if (.rerun | !exist_) {
    obj_ <- .expr
    fst::write_fst(obj_, fil_, 100)
  } else {
    obj_ <- fst::read_fst(fil_)
  }

  return(obj_)
}

lfc <- function(.dirs, .reg = NULL, .rec = FALSE) {
  fils <- unlist(purrr::map(.dirs, ~ list.files(.x, .reg, FALSE, TRUE, .rec)))
  names(fils) <- stringi::stri_replace_all_fixed(
    str = basename(fils),
    pattern = paste0(".", tools::file_ext(fils)),
    replacement = ""
  )
  return(fils)
}

lft <- function(.dirs, .reg = NULL, .id = "doc_id", .rec = FALSE) {
  path <- file_ext <- id <- NULL
  purrr::map_dfr(
    .x = .dirs,
    .f = ~ tibble::tibble(path = list.files(.x, .reg, FALSE, TRUE, .rec))
  ) %>%
    dplyr::mutate(
      file_ext = paste0(".", tools::file_ext(path)),
      id = stringi::stri_replace_last_fixed(basename(path), file_ext, ""),
      path = purrr::set_names(path, id)
    ) %>%
    dplyr::select(id, file_ext, path) %>%
    `colnames<-`(c(.id, "file_ext", "path"))
}
```

# Clean Individual Raw Data
```{r}
get_info_map <- function(.path, .id, .type = c("firm", "individual")) {
  html_raw <- xml2::read_html(.path) %>% rvest::html_nodes(".xdata_block")
  purrr::map_dfr(html_raw, ~ get_info(.x, .id, .type))
}

get_info <- function(.html, .id, .type = c("firm", "individual")) {
  html_div <- rvest::html_children(.html)
  i_html_text <- rvest::html_text(html_div)
  check <- grepl("No license information has been reported", i_html_text)

  if (any(check)) {
    info <- tibble::tibble(
      id = .id,
      info_available = FALSE
    )
  } else {
    if (!length(html_div) == 4) {
      stop("Expected 4 DIVs", call. = FALSE)
    }

    # DIV[[1]] ----------------------------------------------------------------
    name <- rvest::html_text(html_div[[1]])
    name <- trimws(gsub("NAME: ", "", name, fixed = TRUE))

    if (!length(name) == 1) {
      stop("Expected 1 entry in DIV[[1]]", call. = FALSE)
    }

    # DIV[[2]] ----------------------------------------------------------------
    state <- html_div[[2]] %>%
      rvest::html_nodes("a") %>%
      rvest::html_text()
    state <- gsub("STATE OF LICENSE: ", "", state, fixed = TRUE)
    state <- trimws(gsub(" (State Info)", "", state, fixed = TRUE))

    if (!length(state) == 1) {
      stop("Expected state in DIV[[2]] to be length 1", call. = FALSE)
    }

    update_record <- html_div[[2]] %>%
      rvest::html_nodes("div") %>%
      rvest::html_text()
    if (!length(update_record) %in% 1:3) {
      stop("Expected update/record in DIV[[2]] to be length 2 or 3", call. = FALSE)
    }

    update <- trimws(gsub("Record Last Updated: ", "", update_record[1], fixed = TRUE))

    if (length(update_record) == 2) {
      record <- update_record[2]
    } else {
      record <- NULL
    }

    if (length(update_record) == 3) {
      record_info <- update_record[3]
    } else {
      record_info <- NULL
    }


    # DIV[[3]] ----------------------------------------------------------------
    info1 <- html_div[[3]] %>%
      as.character() %>%
      gsub("<br>", "; ", .) %>%
      xml2::read_html() %>%
      rvest::html_node("table") %>%
      rvest::html_table(fill = TRUE) %>%
      tibble::as_tibble() %>%
      dplyr::select(-X1)
    info1 <- info1[, purrr::map_lgl(info1, ~ !all(is.na(.x)))]

    if (ncol(info1) == 0) {
      info1 <- tibble::tibble(address_mail = NA_character_)
    } else {
      if (nrow(info1) == 1) {
        for (i in 1:ncol(info1)) {
          if (grepl("^Business", info1[[i]])) {
            info1[, i] <- gsub("^Business;", "", info1[[i]])
            colnames(info1)[i] <- "address_business"
          } else {
            info1[, i] <- gsub("^Mail;", "", info1[[i]])
            colnames(info1)[i] <- "address_mail"
          }
        }
      } else {
        info1 <- info1 %>%
          dplyr::mutate(dplyr::across(
            .cols = dplyr::everything(),
            .fns = ~ dplyr::if_else(dplyr::row_number() == 1, paste("address", .), .)
          )) %>%
          first_row_to_colnames() %>%
          janitor::clean_names()
      }
    }

    # DIV[[4]] ----------------------------------------------------------------
    info2 <- html_div[[4]] %>%
      rvest::html_node("table") %>%
      rvest::html_table(fill = TRUE) %>%
      tibble::as_tibble() %>%
      dplyr::filter(!endsWith(X1, "Click here for contact information.")) %>%
      dplyr::filter(!X1 == "") %>%
      dplyr::mutate(
        X2 = trimws(gsub("(Details)", "", X2, fixed = TRUE)),
        X2 = trimws(gsub("(Definition)", "", X2, fixed = TRUE))
      ) %>%
      tidyr::pivot_wider(names_from = X1, values_from = X2) %>%
      janitor::clean_names()

    info <- dplyr::bind_cols(info1, info2) %>%
      dplyr::mutate(
        name = name, record = record, state = state, update = update,
        record_info = record_info, id = .id
      ) %>%
      dplyr::select(id, dplyr::everything()) %>%
      dplyr::mutate(dplyr::across(dplyr::everything(), as.character)) %>%
      dplyr::mutate(info_available = TRUE)
    info[info == ""] <- NA_character_
  }

  if (.type == "firm") {
    dplyr::rename(info, firm_id = id)
  } else {
    dplyr::rename(info, individual_id = id)
  }
}

clean_tables_raw <- function(.tab) {
  .tab %>%
    dtplyr::lazy_dt() %>%
    dplyr::select(
      ind_id,
      license_num,
      state = jurisdiction,
      license_status,
      last_name,
      middle_name,
      first_name,
      action
    ) %>%
    dplyr::distinct() %>%
    dplyr::arrange(ind_id) %>%
    dplyr::mutate(
      entry_id = dplyr::row_number(),
      dplyr::across(c(last_name, middle_name, first_name), ~ trimws(gsub("\\s+", " ", .)))
    ) %>%
    dplyr::group_by(ind_id) %>%
    dplyr::mutate(
      n_last_name = dplyr::n_distinct(last_name),
      n_middle_name = dplyr::n_distinct(middle_name),
      n_first_name = dplyr::n_distinct(first_name)
    ) %>%
    dplyr::ungroup() %>%
    tibble::as_tibble()
}

clean_htmls_raw <- function(.tab) {
  tab_ind_htmls_raw %>%
    dtplyr::lazy_dt() %>%
    dplyr::rename(ind_id = individual_id) %>%
    dplyr::mutate(
      ind_id = as.integer(gsub(".+?_", "", ind_id)),
      name = trimws(gsub("\\s+", " ", name))
    ) %>%
    dplyr::mutate(record = as.integer(stringi::stri_extract_first_regex(record, "\\d+"))) %>%
    dplyr::select(
      ind_id,
      license_num = license_permit_certificate_number, state,
      license_status = license_permit_certificate_status,
      full_name = name, action = enforcement_non_compliance_or_disciplinary_actions_details,
      record, cpe_id, issue_date, expiration_date, updated = update,
      license_type, license_basis = basis_for_license,
      registration_num = registration_number,
      address_mail, address_business, can_practice = cpa_can_practice_public_accounting,
      record_info, other_info = other_information
    ) %>%
    dplyr::mutate(dplyr::across(c(issue_date, expiration_date, updated), ymd)) %>%
    dplyr::arrange(dplyr::desc(updated), record) %>%
    dplyr::distinct(dplyr::across(!dplyr::matches("^record$|^updated$")), .keep_all = TRUE) %>%
    dplyr::mutate(entry_id = dplyr::row_number()) %>%
    dplyr::group_by(ind_id) %>%
    dplyr::mutate(n_full_name = dplyr::n_distinct(full_name)) %>%
    dplyr::ungroup() %>%
    tibble::as_tibble()
}

merge_tables_and_htmls <- function(.tab_tables, .tab_htmls) {
  .tab_tables %>%
    dtplyr::lazy_dt() %>%
    dplyr::left_join(
      y  = .tab_htmls,
      by = c("ind_id", "license_num", "state", "license_status", "action")
    ) %>%
    dplyr::arrange(ind_id, license_num, state) %>%
    dplyr::distinct() %>%
    dplyr::mutate(
      dplyr::across(c(address_mail, address_business), ~ stringi::stri_replace_all_fixed(., full_name, "")),
      mult_detailed_info = as.integer(duplicated(entry_id.x) | duplicated(entry_id.x, fromLast = TRUE))
    ) %>%
    dplyr::group_by(ind_id) %>%
    dplyr::mutate(entry_id = row_number()) %>%
    dplyr::ungroup() %>%
    dplyr::select(
      ind_id,
      entry_id,
      cpe_id,
      state,
      issue_date,
      expiration_date,
      full_name,
      last_name,
      middle_name,
      first_name,
      license_num,
      license_status,
      license_type,
      license_basis,
      action, dplyr::everything(), -entry_id.x, -entry_id.y
    ) %>%
    tibble::as_tibble()
}
```

# Get Location Data 
```{r}
prep_address_table <- function(.tab_ind) {
  dplyr::select(.tab_ind, address_business, address_mail) %>%
    tidyr::pivot_longer(dplyr::everything(), names_to = "address_type", values_to = "address_raw") %>%
    dplyr::filter(!is.na(address_raw)) %>%
    dplyr::mutate(
      address_adj = address_raw %>%
        gsub(".*;", "", .) %>%
        gsub(",$", "", .) %>%
        trimws()
    ) %>%
    tidyr::separate(address_adj, c("city", "state", "country"), sep = ",", remove = FALSE) %>%
    dplyr::mutate(dplyr::across(c(city, state, country), trimws)) %>%
    dplyr::mutate(state = dplyr::case_when(
      state == "MINN" ~ "MN",
      TRUE ~ state
    )) %>%
    dplyr::mutate(state = toupper(trimws(gsub("\\d", "", state)))) %>%
    dplyr::mutate(address_adj = stringi::stri_replace_all_fixed(address_adj, "ST.", "Saint")) %>%
    dplyr::distinct(address_type, address_raw, .keep_all = TRUE) %>%
    dplyr::filter(!address_adj == "", !is.na(address_adj))
}

get_hud_data <- function(.paths) {
  lst_hud <- purrr::map(
    .x = .paths,
    .f = ~ openxlsx::read.xlsx(.x) %>%
      tibble::as_tibble() %>%
      janitor::clean_names() %>%
      dplyr::group_by(zip) %>%
      dplyr::arrange(dplyr::desc(res_ratio)) %>%
      dplyr::slice(1) %>%
      dplyr::ungroup() %>%
      dplyr::select(-c(res_ratio:tot_ratio)),
    .progress = TRUE
  )

  tab_hud <- purrr::reduce(lst_hud, full_join, by = "zip")
  names(tab_hud) <- c(names(tab_hud)[1], paste0(names(tab_hud)[-1], "_hud"))
  return(tab_hud)
}

get_dol_data <- function(.paths) {
  tab_dol <- openxlsx::read.xlsx(.paths, startRow = 11) %>%
    janitor::clean_names() %>%
    dplyr::rename(zip = zip_code) %>%
    dplyr::select(-state, -x9)
  names(tab_dol) <- c(names(tab_dol)[1], paste0(names(tab_dol)[-1], "_dol"))
  return(tab_dol)
}
```

# Get ZIP Codes
```{r}
usps_zip <- function(.tab_row, rsc, .wait = 0) {
  rsc$navigate("https://tools.usps.com/zip-code-lookup.htm?bycitystate")
  form_city <- rsc$findElement(using = "id", value = "tCity-city-state")
  form_city$clearElement()
  form_city$sendKeysToElement(list(.tab_row$city_name[1]))

  state_num <- which(vec_states == .tab_row$state_code[1]) + 1
  css <- glue::glue("#tState-city-state > option:nth-child({state_num})")
  form_state <- rsc$findElement(using = "css selector", value = css)
  form_state$clickElement()

  button_find <- rsc$findElement(using = "id", value = "zip-by-city-and-state")
  button_find$clickElement()

  Sys.sleep(.wait)

  write(rsc$getPageSource()[[1]], file.path(.dir_store, paste0(.tab_row$id[1], ".html")))
}

usps_zip_map <- function(.port, .lst, .wait = 0) {
  library(RFsel)
  rsc <- get_rsc(.port)
  purrr::walk(.lst, ~ usps_zip(.x, rsc, .wait))
}

extract_zip <- function(.path) {
  html <- read_html(.path)
  city_state <- html %>%
    rvest::html_nodes(xpath = '//*[@id="zip-lookup-app"]/div/div[7]/div/div/div/div[1]/div[2]/ul/li[2]') %>%
    rvest::html_text()

  zip <- html %>%
    rvest::html_nodes("#zipByCityStateDiv") %>%
    rvest::html_text()

  tibble::tibble(
    city_state = city_state,
    zip = zip
  ) %>%
    dplyr::mutate(zip = stringi::stri_extract_all_regex(zip, "\\b\\d+\\b")) %>%
    tidyr::unnest(zip)
}

zip_get_zip_code_info <- function(.tmp_usps, .tab_dol, .tab_hud) {
  tab_zip <- zipcodeR::zip_code_db %>%
    dplyr::select(zip = zipcode, population) %>%
    dplyr::distinct(zip, .keep_all = TRUE)

  tab_usps <- .tmp_usps %>%
    dplyr::filter(!is.na(zip)) %>%
    dplyr::select(id, zip) %>%
    dplyr::left_join(tab_zip, by = "zip") %>%
    dplyr::arrange(id, dplyr::desc(population)) %>%
    dplyr::distinct(id, .keep_all = TRUE) %>%
    dplyr::select(-population)

  tab_zipcode <- zipcodeR::zip_code_db %>%
    dplyr::select(zip = zipcode, major_city, state, population) %>%
    dplyr::mutate(
      major_city = stringr::str_to_title(major_city),
      id = paste0(major_city, "_", state)
    ) %>%
    dplyr::arrange(id, dplyr::desc(population)) %>%
    dplyr::distinct(id, .keep_all = TRUE) %>%
    dplyr::select(id, zip)

  dplyr::bind_rows(tab_usps, tab_zipcode) %>%
    dplyr::distinct(id, .keep_all = TRUE) %>%
    dplyr::left_join(.tab_dol, by = "zip") %>%
    dplyr::left_join(.tab_hud, by = "zip") %>%
    dplyr::left_join(zipcodeR::zip_code_db, by = c("zip" = "zipcode"))
}

zip_merge_ind_with_location <- function(.tab_ind_raw, .tab_merge_adress, .tab_merge_city_state) {
  tab_ind_raw %>%
    dplyr::select(ind_id, entry_id, dplyr::matches("address")) %>%
    dplyr::left_join(
      y = .tab_merge_adress,
      by = c("address_business" = "address_raw"),
      suffix = c("_business", "_mail")
    ) %>%
    dplyr::left_join(
      y = .tab_merge_adress,
      by = c("address_mail" = "address_raw"),
      suffix = c("_business", "_mail")
    ) %>%
    dplyr::mutate(
      has_address_business = as.integer(!is.na(address_business)),
      has_address_mail = as.integer(!is.na(address_mail)),
      has_all_address = as.integer(!is.na(address_business) & !is.na(address_mail)),
      same_address = as.integer(
        city_name_mail == city_name_business & state_code_mail == state_code_business
      )
    ) %>%
    tibble::as_tibble() %>%
    dplyr::mutate(
      address_type = dplyr::case_when(
        !is.na(address_mail) ~ "mail",
        !is.na(address_business) ~ "business"
      ),
      address = dplyr::case_when(
        !is.na(address_mail) ~ address_mail,
        !is.na(address_business) ~ address_business
      )
    ) %>%
    dplyr::left_join(.tab_merge_adress, c("address" = "address_raw")) %>%
    dplyr::left_join(.tab_merge_city_state, by = c("city_name", "state_code")) %>%
    tibble::as_tibble()
}
```

# Get Gender
```{r}
gender_prepare_name_table <- function(.tab) {
  .tab %>%
    dtplyr::lazy_dt() %>%
    dplyr::select(ind_id, first_name, middle_name, issue_date) %>%
    dplyr::arrange(ind_id, issue_date) %>%
    dplyr::group_by(ind_id) %>%
    dplyr::mutate(issue_date = dplyr::first(issue_date)) %>%
    dplyr::ungroup() %>%
    dplyr::distinct() %>%
    tibble::as_tibble() %>%
    dplyr::mutate(
      first_name = first_name %>%
        gsub("[[:punct:]]", " ", .) %>%
        gsub("([[:space:]]|[[:blank:]])+", " ", .) %>%
        trimws(.) %>%
        stringr::str_to_title(.),
      middle_name = middle_name %>%
        gsub("[[:punct:]]", " ", .) %>%
        gsub("([[:space:]]|[[:blank:]])+", " ", .) %>%
        trimws(.) %>%
        stringr::str_to_title(.),
      dplyr::across(c(first_name, middle_name), ~ dplyr::if_else(nchar(.) < 2, NA_character_, .))
    ) %>%
    tidyr::pivot_longer(c(first_name, middle_name), names_to = "name_type", values_to = "name") %>%
    dtplyr::lazy_dt() %>%
    dplyr::filter(!is.na(name)) %>%
    dplyr::distinct(ind_id, issue_date, name) %>%
    tibble::as_tibble() %>%
    tidytext::unnest_tokens(name, name, to_lower = FALSE) %>%
    dtplyr::lazy_dt() %>%
    dplyr::filter(nchar(name) > 1) %>%
    dplyr::distinct() %>%
    dplyr::mutate(
      year_min = lubridate::year(issue_date) - 50,
      year_max = lubridate::year(issue_date) - 30
    ) %>%
    dplyr::select(-issue_date) %>%
    tibble::as_tibble()
}

gender_genderize <- function(.names, .path_cache) {
  pia_regions <- RFsel::pia_get_regions()
  if (!file.exists(.path_cache)) {
    names_prc <- character()
    tab_prc <- tibble()
  } else {
    tab_prc <- read_rds(.path_cache)
    names_prc <- tab_prc[["name"]]
  }
  names_use <- .names[!.names %in% names_prc]

  while (length(names_use) > 0) {
    if (!file.exists(.path_cache)) {
      names_prc <- character()
      tab_prc <- tibble()
    } else {
      tab_prc <- read_rds(.path_cache)
      names_prc <- tab_prc[["name"]]
    }
    names_use <- .names[!.names %in% names_prc]

    RFsel::pia_set_region(region = sample(pia_regions, 1))
    Sys.sleep(10)
    tmp <- try(gender::gender(na.omit(names_use[1:900]), method = "genderize"), silent = TRUE)

    if (inherits(tmp, "try-error")) {
      RFsel::pia_set_region(region = sample(pia_regions, 1))
      Sys.sleep(10)
    } else {
      tab <- bind_rows(tab_prc, tmp)
      write_rds(tab, .path_cache)
    }
  }

  tab <- read_rds(.path_cache)

  return(tab)
}

gender_year <- function(.lst, .method) {
  itab <- suppressWarnings(
    gender::gender(
      names = .lst[["name"]],
      years = .lst[["years"]],
      method = .method,
      countries = "United States"
    )
  )

  if (nrow(itab) == 0) {
    itab <- add_row(itab, name = .lst[["name"]])
  }
  return(itab)
}

gender_get_table_by_method <- function(
    .tab_gender_names, .tab_gender_genderize_name, .tab_gender_ipums_name,
    .tab_gender_ssa_name, .tab_gender_ipums_year, .tab_gender_ssa_year) {
  tmp_gender_name <- dplyr::bind_rows(
    .tab_gender_genderize_name,
    .tab_gender_ipums_name,
    .tab_gender_ssa_name
  ) %>%
    dplyr::select(name, prob_male = proportion_male, method) %>%
    dplyr::filter(!is.na(prob_male))

  tmp_gender_year <- dplyr::bind_rows(
    .tab_gender_ipums_year,
    .tab_gender_ssa_year
  ) %>%
    dplyr::select(id, prob_male = proportion_male, method) %>%
    dplyr::filter(!is.na(prob_male))

  dplyr::bind_rows(
    dplyr::left_join(.tab_gender_names, tmp_gender_name, by = "name"),
    dplyr::left_join(.tab_gender_names, tmp_gender_year, by = "id")
  ) %>%
    dtplyr::lazy_dt() %>%
    dplyr::arrange(ind_id) %>%
    dplyr::group_by(ind_id, method) %>%
    dplyr::summarise(prediction = mean(prob_male, na.rm = TRUE), .groups = "drop") %>%
    dplyr::mutate(
      guess_diff = abs(1 - prediction - prediction),
      gender = dplyr::case_when(
        prediction > .5 ~ "male",
        prediction < .5 ~ "female",
        TRUE ~ "unknown"
      )
    ) %>%
    tibble::as_tibble()
}

```

# Get Race
```{r}
race_prepare_ind_names <- function(.tab_ind_raw, .tab_zip_info, .tab_ind_loc, .tab_gender) {
  
  tab_race_name <- .tab_ind_raw %>%
    dplyr::select(ind_id, state, surname = last_name, issue_date) %>%
    dplyr::mutate(
      surname = surname %>%
        gsub("[[:punct:]]", " ", .) %>%
        gsub("([[:space:]]|[[:blank:]])+", " ", .) %>%
        trimws(.) %>%
        stringr::str_to_title(.)
    ) %>%
    dtplyr::lazy_dt() %>%
    dplyr::group_by(ind_id) %>%
    dplyr::arrange(issue_date, .by_group = TRUE) %>%
    dplyr::mutate(issue_date = dplyr::first(issue_date)) %>%
    dplyr::ungroup() %>%
    dplyr::distinct(ind_id, .keep_all = TRUE) %>%
    tibble::as_tibble()
  
  tab_race_zip_info <- .tab_zip_info %>%
    dplyr::select(zip, county = county_no_dol)
  
  tab_race_zip <- .tab_ind_loc %>%
    dplyr::select(ind_id, entry_id, zip) %>%
    dtplyr::lazy_dt() %>%
    dplyr::left_join(
      y = dplyr::select(.tab_ind_raw, ind_id, entry_id, issue_date),
      by = c("ind_id", "entry_id")
    ) %>%
    dplyr::arrange(ind_id, issue_date) %>%
    dplyr::group_by(ind_id) %>%
    dplyr::mutate(issue_date = dplyr::first(issue_date)) %>%
    dplyr::ungroup() %>%
    dplyr::distinct(ind_id, .keep_all = TRUE) %>%
    dplyr::left_join(tab_race_zip_info, by = "zip") %>%
    dplyr::select(ind_id, county) %>%
    dplyr::distinct(ind_id, .keep_all = TRUE) %>%
    tibble::as_tibble()
  
  tab_race_gender <- .tab_gender %>%
    dplyr::select(ind_id, sex = gender) %>%
    dplyr::mutate(sex = dplyr::case_when(
      sex == "male" ~ 1,
      sex == "female" ~ 0
    ))
  
  tab_ind_race <- tab_race_name %>%
    dtplyr::lazy_dt() %>%
    dplyr::left_join(tab_race_zip, by = "ind_id") %>%
    dplyr::left_join(tab_race_gender, by = "ind_id") %>%
    dplyr::mutate(age = 2021 - lubridate::year(issue_date) + 30) %>%
    tibble::as_tibble()
  
  tab_ind_race[is.na(tab_ind_race)] <- NA
  tab_ind_race[tab_ind_race == -Inf] <- NA
  tab_ind_race[tab_ind_race == Inf] <- NA
  
  return(tab_ind_race)
}

race_wru_surname <- function(.tab) {
  wru::predict_race(
    voter.file = .tab,
    surname.only = TRUE,
    retry = 5
  )
}

race_wru_county <- function(.tab) {
  wru::predict_race(
    voter.file = .tab,
    retry = 5,
    census.geo = "county",
    census.key = "8d569f4a70e1abf84cd23350522b479216d008c4"
  )
}

race_wru_county_age <- function(.tab) {
  wru::predict_race(
    voter.file = .tab,
    retry = 5,
    census.geo = "county",
    census.key = "8d569f4a70e1abf84cd23350522b479216d008c4",
    age = TRUE
  )
}

race_wru_county_sex <- function(.tab) {
  wru::predict_race(
    voter.file = .tab,
    retry = 5,
    census.geo = "county",
    census.key = "8d569f4a70e1abf84cd23350522b479216d008c4",
    sex = TRUE
  )
}

race_wru_county_age_sex <- function(.tab) {
  wru::predict_race(
    voter.file = .tab,
    retry = 5,
    census.geo = "county",
    census.key = "8d569f4a70e1abf84cd23350522b479216d008c4",
    age = TRUE,
    sex = TRUE
  )
}

race_get_predictions <- function(.tab) {
  .tab %>%
    dtplyr::lazy_dt() %>%
    dplyr::select(ind_id, method, starts_with("pred")) %>%
    dplyr::filter(!is.na(method)) %>%
    dplyr::group_by(ind_id, method) %>%
    dplyr::summarise(dplyr::across(dplyr::everything(), ~ mean(., na.rm = TRUE)), .groups = "drop") %>%
    tibble::as_tibble()
  tidyr::pivot_longer(dplyr::starts_with("pred")) %>%
    dtplyr::lazy_dt() %>%
    dplyr::group_by(ind_id, method) %>%
    dplyr::arrange(ind_id, method, desc(value), .by_group = TRUE) %>%
    dplyr::mutate(
      race = dplyr::first(name),
      prediction = value,
      guess_diff = value[1] - value[2]
    ) %>%
    dplyr::slice(1) %>%
    dplyr::ungroup() %>%
    tibble::as_tibble() %>%
    dplyr::select(ind_id, method, race, prediction, guess_diff) %>%
    dplyr::mutate(race = stringi::stri_replace_all_fixed(
      race,
      c("pred.whi", "pred.bla", "pred.his", "pred.asi", "pred.oth"),
      c("white", "black", "hispanic", "asian", "other"),
      vectorize_all = FALSE
    ))
}

race_get_table_by_method <- function(.lst_race, .tab_ind_race, .path_race_census, .path_race_floter) {
  tmp <- dtplyr::lazy_dt(dplyr::select(.tab_ind_race, ind_id, state, surname, age, sex, county))
  
  tab_race_census <- rio::import(.path_race_census) %>%
    dplyr::select(
      ind_id, race,
      prediction = probability, guess_diff = guessdiff,
      method = predictionmethod
    ) %>%
    dplyr::mutate(race = tolower(race))
  
  tab_race_floter <- rio::import(.path_race_floter) %>%
    dplyr::select(
      ind_id, race,
      prediction = probability, guess_diff = guessdiff,
      method = predictionmethod
    ) %>%
    dplyr::mutate(race = tolower(race))
  
  dplyr::bind_rows(
    tibble::as_tibble(dplyr::left_join(tmp, .lst_race$prr_surname, by = c("surname" = "name"))),
    tibble::as_tibble(dplyr::left_join(tmp, .lst_race$wru_surname, by = c("surname", "state"))),
    tibble::as_tibble(dplyr::left_join(tmp, .lst_race$wru_county, by = c("surname", "state", "county"))),
    tibble::as_tibble(dplyr::left_join(tmp, .lst_race$wru_county_age, by = c("surname", "state", "county", "age"))),
    tibble::as_tibble(dplyr::left_join(tmp, .lst_race$wru_county_sex, by = c("surname", "state", "county", "sex"))),
    tibble::as_tibble(dplyr::left_join(tmp, .lst_race$wru_county_age_sex, by = c("surname", "state", "county", "age", "sex")))
  ) %>%
    dtplyr::lazy_dt() %>%
    dplyr::select(ind_id, method, dplyr::starts_with("pred")) %>%
    dplyr::arrange(ind_id) %>%
    dplyr::filter(!is.na(pred.whi)) %>%
    tidyr::pivot_longer(starts_with("pred"), names_to = "race", values_to = "prediction") %>%
    dplyr::mutate(race = stringi::stri_replace_all_fixed(
      race,
      c("pred.whi", "pred.bla", "pred.his", "pred.asi", "pred.oth"),
      c("white", "black", "hispanic", "asian", "other"),
      vectorize_all = FALSE
    )) %>%
    dplyr::group_by(ind_id, method) %>%
    dplyr::arrange(ind_id, method, desc(prediction), .by_group = TRUE) %>%
    dplyr::mutate(
      race = first(race),
      guess_diff = prediction[1] - prediction[2]
    ) %>%
    dplyr::ungroup() %>%
    tibble::as_tibble() %>%
    dplyr::bind_rows(tab_race_census, tab_race_floter)
}
```

# Build Dictionary
```{r}
dict_extract_infos_raw <- function(.html) {
  info <- rvest::read_html(.html) %>%
    rvest::html_nodes("a") %>%
    xml2::xml_attr("href")
  info <- info[startsWith(info, "javascript:windowOpener1")]
  info <- gsub("javascript:windowOpener1(", "", info, fixed = TRUE)
  info <- gsub(")", "", info, fixed = TRUE)
  tibble::tibble(info = info)
}

dict_s_extract_infos_raw <- purrr::safely(dict_extract_infos_raw)

dict_adjust_infos <- function(.tab) {
  .tab %>%
    dplyr::distinct() %>%
    dplyr::mutate(info = stri_split_fixed(info, "','")) %>%
    dplyr::filter(lengths(info) == 4) %>%
    tidyr::unnest_wider(info, names_sep = "_") %>%
    dplyr::select(type_raw = info_4, state = info_1, var = info_2, info = info_3) %>%
    dplyr::mutate(dplyr::across(dplyr::everything(), ~ trimws(gsub("'", "", ., fixed = TRUE)))) %>%
    dplyr::mutate(var = if_else(type_raw == "CPE-ID", "CPE-ID", var)) %>%
    dplyr::distinct() %>%
    dplyr::arrange(type_raw, state, var) %>%
    dplyr::mutate(type_adj = dplyr::case_when(
      type_raw == "License/Permit/Certificate Status" ~ "license_status",
      type_raw == "License Type" ~ "license_type",
      type_raw == "CPE-ID" ~ "cpe_id",
      type_raw == "Basis for License" ~ "license_basis"
    )) %>%
    dplyr::relocate(type_adj, .after = type_raw) %>%
    dplyr::mutate(
      info = gsub("<br/>", " ", info, fixed = TRUE),
      info = gsub("\\s+", " ", info)
    )
}
```
