## ----setup, include=FALSE-----------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.width = 6,
  fig.height = 4,
  dpi = 110
)

## ----data---------------------------------------------------------------------
asah <- pROC::aSAH
str(asah[, c("outcome", "gos6", "gender", "age", "wfns", "s100b", "ndka")])
table(asah$outcome)

## ----leakage------------------------------------------------------------------
table(asah$gos6, asah$outcome)

## ----spec---------------------------------------------------------------------
library(uniml)

spec <- uniml_spec(
  task_type = "binary",
  outcome = "outcome",
  positive = "Poor",
  exclude = "gos6"
)
spec <- set_preprocess(
  spec,
  impute = "median_mode",
  scale = "zscore",
  encode = "onehot"
)
spec <- set_feature_selection(
  spec,
  methods = "univariate",
  p_threshold = 0.2
)
spec <- set_learners(spec, c("lr", "cart"), tune = FALSE)
spec <- set_tuning(spec, folds = 5, repeats = 5, evals = 4, metric = "auc", seed = 123)
spec <- set_evaluation(
  spec,
  ci = "bootstrap",
  calibration = "platt",
  threshold_method = "youden",
  select_on = "validation"
)
spec <- set_leakage_policy(spec, mode = "strict_in")
spec <- set_explain(spec, model = "none", plots = character())
spec

## ----split--------------------------------------------------------------------
split <- make_split(
  as_uniml_data(asah, spec),
  prop = c(0.7, 0.3),
  stratify = TRUE,
  seed = 123
)

data.frame(
  set = c("train", "validation"),
  n = c(nrow(split$train$data), nrow(split$validation$data)),
  poor_outcome = c(
    sum(split$train$data$outcome == "Poor"),
    sum(split$validation$data$outcome == "Poor")
  )
)

## ----run----------------------------------------------------------------------
result <- suppressWarnings(run_workflow(
  spec,
  train = split$train$data,
  validation = split$validation$data,
  ci_R = 40
))
result

## ----failures-----------------------------------------------------------------
result$meta$failures

## ----cv-----------------------------------------------------------------------
cv_auc <- subset(result$cv$summary, metric == "auc")
cv_auc[, c("model", "mean", "sd", "lower", "upper", "n")]

## ----metrics------------------------------------------------------------------
validation <- subset(
  result$metrics,
  set == "validation",
  select = c("model", "probability", "auc", "brier", "sensitivity",
             "specificity", "ppv", "npv")
)
validation

## ----ci-----------------------------------------------------------------------
ci <- subset(
  result$ci,
  set == "validation" & metric %in% c("auc", "brier"),
  select = c("model", "probability", "metric", "mean", "lower", "upper")
)
ci

## ----ci-undefined-------------------------------------------------------------
subset(
  result$ci,
  set == "validation" & metric %in% c("plr", "dor"),
  select = c("model", "metric", "mean", "lower", "upper", "n_effective")
)

## ----ranking------------------------------------------------------------------
result$ranking[, c("model", "score", "rank", "metric", "set")]

## ----roc, fig.cap = "ROC curves on the held-out validation patients."---------
plot_roc(result, set = "validation")

## ----calibration, fig.cap = "Calibration on the validation set, before and after Platt scaling."----
plot_calibration(result, set = "validation")

## ----dca, fig.cap = "Decision curve analysis."--------------------------------
plot_dca(result)

## ----confusion, fig.cap = "Confusion matrix at the out-of-fold Youden threshold."----
plot_confusion(result, models = "best")

## ----thresholds---------------------------------------------------------------
as.data.frame(result$meta$thresholds)

## ----delong-------------------------------------------------------------------
result$tests$delong$summary
result$tests$delong$formatted

## ----hl-----------------------------------------------------------------------
for (track in c("raw", "calibrated")) {
  for (model in c("lr", "cart")) {
    item <- result$tests$hl[[track]][[model]]
    cat(sprintf(
      "%-10s %-5s statistic = %s, df = %s, p = %s (%s)\n",
      track, model,
      format(round(item$statistic, 3)), format(item$df),
      format(round(item$p_value, 3)), item$method
    ))
  }
}

## ----nri----------------------------------------------------------------------
subset(
  result$tests$nri_idi,
  set == "validation",
  select = c("model", "reference", "nri", "nri_events", "nri_nonevents", "idi")
)

## ----tripod-------------------------------------------------------------------
checklist <- tripod_ai_check(result)
table(checklist$status)
subset(checklist, status != "reported", select = c("domain", "status"))

## ----export-------------------------------------------------------------------
out <- tempfile("uniml-asah-")
exported <- export_results(result, dir = out, include_plots = FALSE)
basename(exported$files)

