## ----include = FALSE----------------------------------------------------------
# Evaluate chunks only where the ducklake DuckDB extension is already
# installed. The probe never downloads anything, so building this vignette
# needs no network access.
ducklake_available <- ducklake::ducklake_extension_available()
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  eval = ducklake_available
)
# Use a unique temp directory for this vignette to avoid conflicts during R CMD check
vignette_temp_dir <- file.path(tempdir(), "storage_backups_vignette")
dir.create(vignette_temp_dir, showWarnings = FALSE, recursive = TRUE)
knitr::opts_knit$set(root.dir = vignette_temp_dir)

## ----setup, message=FALSE-----------------------------------------------------
library(ducklake)
library(dplyr)
library(fs)

## ----storage-patterns, eval=FALSE---------------------------------------------
# # Local storage - fastest, but not shared
# attach_ducklake(
#   ducklake_name = "local_lake",
#   lake_path = "~/data/my_ducklake"
# )
# 
# # PostgreSQL catalog with S3 data - multi-client, scalable
# attach_ducklake(
#   ducklake_name = "shared_lake",
#   backend = "postgres",
#   catalog_connection_string = "dbname=ducklake_catalog host=localhost",
#   lake_path = "s3://my-bucket/ducklake/data"
# )
# 
# # SQLite catalog - lightweight multi-client option
# attach_ducklake(
#   ducklake_name = "team_lake",
#   backend = "sqlite",
#   catalog_connection_string = "~/data/metadata.sqlite",
#   lake_path = "~/data/parquet_files"
# )

## ----storage-secrets, eval=FALSE----------------------------------------------
# # Explicit keys, scoped to one bucket
# create_storage_secret(
#   "s3",
#   key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
#   secret = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
#   region = "us-east-1",
#   scope = "s3://my-bucket"
# )
# 
# # Or let the AWS credential chain find them (env vars, profiles,
# # instance metadata) -- no keys in code
# create_storage_secret("s3", provider = "credential_chain")
# 
# # Then attach as usual
# attach_ducklake("shared_lake", lake_path = "s3://my-bucket/ducklake/data")

## ----create-sample-lake-------------------------------------------------------
# Create a temporary directory for our demo
lake_dir <- file.path(vignette_temp_dir, "storage_demo")
dir.create(lake_dir, showWarnings = FALSE, recursive = TRUE)

# Install ducklake extension
# The ducklake extension only needs installing once per machine:
# install_ducklake()

# Create and populate a DuckLake
attach_ducklake(
  ducklake_name = "demo_lake",
  lake_path = lake_dir
)

# Add some data with transactions
with_transaction(
  create_table(mtcars[1:15, ], "cars"),
  author = "Demo User",
  commit_message = "Initial load"
)

with_transaction(
  get_ducklake_table("cars") |>
    mutate(hp_per_cyl = hp / cyl) |>
    replace_table("cars"),
  author = "Demo User",
  commit_message = "Add hp_per_cyl metric"
)

with_transaction(
  get_ducklake_table("cars") |>
    mutate(mpg_adjusted = if_else(cyl == 4, mpg * 1.1, mpg)) |>
    replace_table("cars"),
  author = "Demo User",
  commit_message = "Add adjusted MPG for 4-cylinder cars"
)

## ----view-catalog-files-------------------------------------------------------
dir_tree(lake_dir)

## ----view-data-files----------------------------------------------------------
# Data files are organized by schema and table
main_dir <- file.path(lake_dir, "main")

dir_tree(main_dir, recurse = 2)
  
# Get details about parquet files
parquet_files <- dir_ls(main_dir, recurse = TRUE, regexp = "\\.parquet$")
for (f in parquet_files) {
  cat(sprintf("  %s (%s bytes)\n", 
              path_file(f), 
              file.size(f)))
}

## ----examine-structure--------------------------------------------------------
# List all snapshots to see the version history
snapshots <- list_table_snapshots("cars")
snapshots |>
  select(snapshot_id, author, commit_message)

## ----backup-catalog-simple----------------------------------------------------
# Create backup directory
backup_dir <- file.path(lake_dir, "backups")
dir.create(backup_dir, showWarnings = FALSE)

# Release file locks before copying the catalog
detach_ducklake("demo_lake", shutdown = TRUE)

# Copy the catalog file to create a backup
file.copy(
  from = file.path(lake_dir, "demo_lake.ducklake"),
  to = file.path(backup_dir, "demo_lake.ducklake")
)

# Copy the data directory as well
dir_copy(
  path = file.path(lake_dir, "main"),
  new_path = file.path(backup_dir, "main")
)

# Verify the backup was created
dir_tree(backup_dir)

# To work with the backup, attach it. override_data_path is needed because
# the catalog remembers the original data location, which the backup no
# longer matches.
attach_ducklake(
  ducklake_name = "demo_lake",
  lake_path = backup_dir,
  override_data_path = TRUE
)

# Verify you're working with the backup
list_table_snapshots("cars")

# You can switch back to the original by detaching and reattaching
detach_ducklake("demo_lake")
attach_ducklake("demo_lake", lake_path = lake_dir)

## ----backup-storage-local, eval=FALSE-----------------------------------------
# # Use file system tools to copy the entire data directory
# backup_data_dir <- file.path(lake_dir, "backups", "main_backup")
# dir_copy(
#   path = file.path(lake_dir, "main"),
#   new_path = backup_data_dir
# )

## ----backup-cloud, eval=FALSE-------------------------------------------------
# # Original
# attach_ducklake(
#   ducklake_name = "prod_lake",
#   lake_path = "s3://original-bucket/data"
# )
# 
# # After recovery from replicated bucket
# attach_ducklake(
#   ducklake_name = "prod_lake",
#   lake_path = "s3://backup-bucket/data",
#   override_data_path = TRUE
# )

## ----recovery-catalog, eval=FALSE---------------------------------------------
# # Restore from backup by copying the backup file
# # (backup_dir here is a directory created earlier, e.g. by backup_ducklake())
# file.copy(
#   from = file.path(backup_dir, "demo_lake.ducklake"),
#   to = file.path(lake_dir, "demo_lake.ducklake"),
#   overwrite = TRUE
# )
# 
# # Reattach to the restored database
# attach_ducklake("demo_lake", lake_path = lake_dir)
# 
# # Verify recovery by listing snapshots
# list_table_snapshots("cars")

## ----recovery-data, eval=FALSE------------------------------------------------
# # Restore data files from backup
# dir_copy(
#   path = backup_data_dir,
#   new_path = file.path(lake_dir, "main"),
#   overwrite = TRUE
# )
# 
# # DuckLake will automatically reconnect to the restored files
# # since the catalog maintains the file paths

## ----routine-maintenance, eval=FALSE------------------------------------------
# # Compact small adjacent Parquet files into larger ones
# merge_adjacent_files()
# 
# # Preview a retention policy, then apply it
# expire_snapshots(older_than = Sys.time() - 30 * 24 * 60 * 60, dry_run = TRUE)
# expire_snapshots(older_than = Sys.time() - 30 * 24 * 60 * 60)
# 
# # Expired snapshots only *schedule* file deletion; this reclaims the storage
# cleanup_old_files(cleanup_all = TRUE)
# 
# # Rewrite data files whose rows have mostly been deleted
# rewrite_data_files(delete_threshold = 0.5)
# 
# # Remove untracked files from the data path -- always dry-run this one first
# delete_orphaned_files(dry_run = TRUE, cleanup_all = TRUE)

## ----maintenance-backup-sequence, eval=FALSE----------------------------------
# # Recommended backup sequence
# 
# # 1. Run maintenance operations (if needed)
# merge_adjacent_files()
# expire_snapshots(older_than = Sys.time() - 30 * 24 * 60 * 60)
# cleanup_old_files(cleanup_all = TRUE)
# 
# # 2. Ensure all transactions are committed
# # (no pending work)
# 
# # 3. Release file locks before copying the catalog
# detach_ducklake("demo_lake", shutdown = TRUE)
# 
# # 4. Back up catalog
# dir.create(file.path(lake_dir, "backups"), showWarnings = FALSE)
# file.copy(
#   from = file.path(lake_dir, "demo_lake.ducklake"),
#   to = file.path(lake_dir, "backups",
#                  paste0("backup_", format(Sys.time(), "%Y%m%d_%H%M%S"), ".ducklake"))
# )
# 
# # 5. Back up data files
# dir_copy(
#   path = file.path(lake_dir, "main"),
#   new_path = file.path(lake_dir, "backups", "main_latest")
# )
# 
# # 6. Re-attach and continue working
# attach_ducklake("demo_lake", lake_path = lake_dir)

## ----complete-backup-function-------------------------------------------------
# Create a complete backup with timestamp
backup_dir <- backup_ducklake(
  ducklake_name = "demo_lake",
  lake_path = lake_dir,
  backup_path = file.path(lake_dir, "backups")
)

# The function returns the backup directory path
print(backup_dir)

## ----cleanup------------------------------------------------------------------
# Detach the demo lake
detach_ducklake("demo_lake")

# Clean up temporary files
unlink(lake_dir, recursive = TRUE)