In this initial step, we load the required R packages and stream in
the primary dataset (pt_data_full.jsonl) generated from the
previous clinical note extraction phase.
library(tidyllm)
library(jsonlite)
library(purrr)
library(dplyr)
# Load data
json_file = "pt_data_full.jsonl"
if (file.exists(json_file)) { # check if we indeed have the file
patient_data <- stream_in(file(json_file)) # Read the jsonl file
glimpse(patient_data) # Glimpse into data
} else {
print("File not found.")
}#> Found 13 records... Imported 13 records. Simplifying...#> Rows: 13
#> Columns: 8
#> $ patient_name <chr> "Ramasamy", "Danial", "Bella", "Amy", "Ryan", "Suhaimi", …
#> $ age <dbl> 45.000, 0.500, 56.000, 35.000, 8.000, 45.000, 27.000, 70.…
#> $ gender <chr> "male", "male", "female", "female", "male", "male", "fema…
#> $ symptoms <list> <"heavy alcohol consumption", "smoker", "bedridden", "ri…
#> $ pe <list> <"cardiomegaly", "hepatomegaly">, NA, NA, NA, <"pale", "…
#> $ ix_blood <list> NA, NA, NA, "HIV reactive", <"TWBC 80 x10^9/l", "Hb 7.8g…
#> $ ix_imaging <list> NA, NA, NA, NA, NA, "KUB radiograph: bilateral staghorn …
#> $ diagnosis <chr> "NA", "beta thalassemia major", "breast cancer", "HIV inf…We inspect the overall dataset structure using an interactive
datatable and construct baseline demographic summary tables with
gtsummary.
# Overview data
glimpse(patient_data)#> Rows: 13
#> Columns: 8
#> $ patient_name <chr> "Ramasamy", "Danial", "Bella", "Amy", "Ryan", "Suhaimi", …
#> $ age <dbl> 45.000, 0.500, 56.000, 35.000, 8.000, 45.000, 27.000, 70.…
#> $ gender <chr> "male", "male", "female", "female", "male", "male", "fema…
#> $ symptoms <list> <"heavy alcohol consumption", "smoker", "bedridden", "ri…
#> $ pe <list> <"cardiomegaly", "hepatomegaly">, NA, NA, NA, <"pale", "…
#> $ ix_blood <list> NA, NA, NA, "HIV reactive", <"TWBC 80 x10^9/l", "Hb 7.8g…
#> $ ix_imaging <list> NA, NA, NA, NA, NA, "KUB radiograph: bilateral staghorn …
#> $ diagnosis <chr> "NA", "beta thalassemia major", "breast cancer", "HIV inf…DT::datatable(patient_data)# Sample data
patient_data$age#> [1] 45.000 0.500 56.000 35.000 8.000 45.000 27.000 70.000 60.000 0.019
#> [11] 50.000 17.000 35.000patient_data$gender#> [1] "male" "male" "female" "female" "male" "male" "female" "male"
#> [9] "male" "female" "female" "male" "male"gtsummary::tbl_summary(patient_data, include = c(age, gender))| Characteristic | N = 131 |
|---|---|
| age | 35 (17, 50) |
| gender | |
| female | 5 (38%) |
| male | 8 (62%) |
| 1 Median (Q1, Q3); n (%) | |
Here, we filter out specific target patients who have recorded blood
investigation notes and assemble a targeted subset
(patient_data_bld).
# Patient's with blood Ix
names_bld = c("Ryan", "Suhaimi", "Fatimah", "Ahmed", "Najeeb")
patient_data[patient_data$patient_name %in% names_bld, "ix_blood"] #> [[1]]
#> [1] "TWBC 80 x10^9/l"
#> [2] "Hb 7.8g/dl"
#> [3] "Platelet 20 x10^9/l"
#> [4] "Blood film leucocytosis with 80% blast cells"
#>
#> [[2]]
#> [1] "Urine dipstick: microscopic haematuria and leukocytes"
#> [2] "Hb: 10.4 g/dL"
#> [3] "TWBC: 21.3 x 109/L"
#> [4] "Potassium: 5.7 mmol/L"
#> [5] "Sodium: 136 mmol/L"
#> [6] "Urea: 27 mmol/L"
#> [7] "Creatinine: 213 µmol/L"
#>
#> [[3]]
#> [1] "Hb: 11.9 g/dl"
#> [2] "WBC: 17.8 x 109/liter"
#> [3] "Neutrophils: 70%"
#> [4] "Lymphocytes: 24%"
#> [5] "Eosinophils: 2%"
#> [6] "Monocytes: 4%"
#> [7] "Platelets: 250 x 109/liter"
#> [8] "Na+: 139 mmol/L"
#> [9] "K+: 4.5 mmol/L"
#> [10] "CI-: 94 mmol/L"
#> [11] "Ca++: 2.1 mmol/L"
#> [12] "PO42-: 1.4 mmol/L"
#> [13] "Urea: 6.5 mmol/L"
#> [14] "Creatinine: 80 µmol/L"
#> [15] "urine culture: growth of E. coli (colony count 105/ml)"
#> [16] "urinalysis: cloudy, pH 5.5, protein +, glucose -, ketone -, bilirubin -, pus cells numerous, Red cells 5-10 / High power field, casts present, epithelial cells seen, bacteria seen"
#>
#> [[4]]
#> [1] "Hb 12.0 g/dL" "TWBC 7 x 10^9/L" "Potassium 5.6 mmol/L"
#> [4] "Sodium 136 mmol/L" "Urea 23 mmol/L" "Creatinine 200 µmol/L"
#> [7] "PSA 2 ng/ml"
#>
#> [[5]]
#> [1] "Hb: 15 g/dl" "PCV: 50%" "Sodium: 140 mmol/L"
#> [4] "Potassium: 4.9 mmol/L" "Urea: 20 mmol/L" "Creatinine: 110 µmol/L"
#> [7] "pH: 7.2" "PO2: 120 mmHg" "PCO2: 28 mmHg"
#> [10] "HCO3-: 16 mmol/L" "BE: -6" "O2 saturation: 98%"patient_data[patient_data$patient_name %in% names_bld, c("patient_name", "age","gender","ix_blood")] -> patient_data_bldWe construct a focused numerical schema (patient_schema)
for extracting discrete laboratory values (Hb, WBC, Sodium, Potassium,
Urea, Creatinine). Numerical missingness is explicitly mapped to a
placeholder value (99999).
# Bld schema
patient_schema <- tidyllm_schema(
name = "Blood Ix",
Hb = field_dbl(
"Hemoglobin level. If this information is not available, output exactly 99999."
),
WBC = field_dbl(
"White cell count. If this information is not available, output exactly 99999."
),
Sodium = field_dbl(
"Sodium. If this information is not available, output exactly 99999."
),
Potassium = field_dbl(
"Potassium. If this information is not available, output exactly 99999."
),
Urea = field_dbl(
"Urea. If this information is not available, output exactly 99999."
),
Creatinine = field_dbl(
"Creatinine. If this information is not available, output exactly 99999."
)
)
# Set output file in JSON Lite format
output_file = "pt_data_bld.jsonl"Clear the output file before starting a fresh run. Otherwise, with multiple run, new data will be added to the existing file
if (file.exists(output_file)) file.remove(output_file)#> [1] TRUE# Get variable ix_blood as text
ix_blood_texts = patient_data_bld$ix_blood |> sapply(paste, collapse = ", ")We send the unnested blood text snippets into Google’s Gemini LLM
using tidyllm, writing structured extraction outputs
row-by-row into a JSON Lines file.
# Process using a tidy pipeline
# Need to iterate as now we have two notes
# API
Sys.setenv(GOOGLE_API_KEY = readLines("api1"))
for(text in ix_blood_texts) {
full_prompt <- paste("Extract blood investigation according to the schema.", text)
result <- llm_message(full_prompt) |>
chat(
.provider = gemini(), # Gemini as LLM provider
.model = "gemini-3.1-flash-lite", # Gemini model
.json_schema = patient_schema # Schema
)
# Extract the textual JSON reply from the LLM message object
json_string <- get_reply(result)
# Minify and append to .jsonl
compact_json <- minify(json_string)
cat(compact_json, "\n", file = output_file, append = TRUE, sep = "")
}Finally, we stream back the extracted laboratory data frame, merge it
with patient demographics, and replace our sentinel numeric placeholder
(99999) with native R NA values.
# Read the file and combine
patient_data_bld_sub <- stream_in(file(output_file))#> opening file input connection.#> Found 5 records... Imported 5 records. Simplifying...#> closing file input connection.patient_data_bld1 <- data.frame(patient_data_bld[,1:3], patient_data_bld_sub)
str(patient_data_bld1)#> 'data.frame': 5 obs. of 9 variables:
#> $ patient_name: chr "Ryan" "Suhaimi" "Fatimah" "Ahmed" ...
#> $ age : num 8 45 27 70 60
#> $ gender : chr "male" "male" "female" "male" ...
#> $ Hb : num 7.8 10.4 11.9 12 15
#> $ WBC : num 80 21.3 17.8 7 99999
#> $ Sodium : num 99999 136 139 136 140
#> $ Potassium : num 99999 5.7 4.5 5.6 4.9
#> $ Urea : num 99999 27 6.5 23 20
#> $ Creatinine : num 99999 213 80 200 110patient_data_bld1#> patient_name age gender Hb WBC Sodium Potassium Urea Creatinine
#> 5 Ryan 8 male 7.8 80.0 99999 99999.0 99999.0 99999
#> 6 Suhaimi 45 male 10.4 21.3 136 5.7 27.0 213
#> 7 Fatimah 27 female 11.9 17.8 139 4.5 6.5 80
#> 8 Ahmed 70 male 12.0 7.0 136 5.6 23.0 200
#> 9 Najeeb 60 male 15.0 99999.0 140 4.9 20.0 110# Replace -999 with true NA
patient_data_bld1 <- patient_data_bld1 |>
mutate(across(c(Hb, WBC, Sodium, Potassium, Urea, Creatinine), ~na_if(.x, 99999)))
patient_data_bld1#> patient_name age gender Hb WBC Sodium Potassium Urea Creatinine
#> 5 Ryan 8 male 7.8 80.0 NA NA NA NA
#> 6 Suhaimi 45 male 10.4 21.3 136 5.7 27.0 213
#> 7 Fatimah 27 female 11.9 17.8 139 4.5 6.5 80
#> 8 Ahmed 70 male 12.0 7.0 136 5.6 23.0 200
#> 9 Najeeb 60 male 15.0 NA 140 4.9 20.0 110of which we can do other statistical analyses