04 · Datenimport aus Dateien und APIs

import pandas as pd

# Standard case: comma-separated, UTF-8.
cohort = pd.read_csv("https://datascience.medizin.dev/data/kohorte.csv")

# German hospital format: semicolon, latin-1, decimal comma.
cohort_de = pd.read_csv(
    "https://datascience.medizin.dev/data/kohorte_de.csv",
    sep=";",
    encoding="latin-1",
    decimal=",",
)

# Load only specific columns (saves memory).
subset = pd.read_csv("https://datascience.medizin.dev/data/kohorte.csv",
                     usecols=["patient_id", "alter", "aufnahmegrund"])

library(readr)

# Standard case.
cohort <- read_csv("https://datascience.medizin.dev/data/kohorte.csv", show_col_types = FALSE)

# German hospital format: semicolon + decimal comma.
cohort_de <- read_delim(
  "https://datascience.medizin.dev/data/kohorte_de.csv",
  delim = ";",
  locale = locale(encoding = "latin1", decimal_mark = ","),
  show_col_types = FALSE
)

# Shorthand for German format.
cohort_de2 <- read_csv2("https://datascience.medizin.dev/data/kohorte_de.csv",
                         locale = locale(encoding = "latin1"),
                         show_col_types = FALSE)

import pandas as pd

# First sheet (default).
table = pd.read_excel("https://datascience.medizin.dev/data/einschluss.xlsx")

# Specific sheet, specific columns.
lab_data = pd.read_excel(
    "https://datascience.medizin.dev/data/einschluss.xlsx",
    sheet_name="Labor",
    usecols="A:F",
)

library(readxl)

tmp <- tempfile(fileext = ".xlsx")
download.file("https://datascience.medizin.dev/data/einschluss.xlsx", tmp, mode = "wb", quiet = TRUE)

# First sheet.
table <- read_excel(tmp)

# Specific sheet.
lab_data <- read_excel(tmp, sheet = "Labor")

import json
import urllib.request
import pandas as pd

# Flat JSON directly into a DataFrame.
df = pd.read_json("https://datascience.medizin.dev/data/beispiel.json")

# Nested JSON: fetch with urllib, then json_normalize.
with urllib.request.urlopen("https://datascience.medizin.dev/data/fhir_bundle.json") as response:
    bundle = json.load(response)
df_fhir = pd.json_normalize(bundle["entry"], sep="_")

library(jsonlite)
library(tibble)

# Flat JSON – fromJSON() reads URLs directly.
df <- fromJSON("https://datascience.medizin.dev/data/beispiel.json")

# Nested JSON -> tibble.
bundle  <- fromJSON("https://datascience.medizin.dev/data/fhir_bundle.json", flatten = TRUE)
df_fhir <- as_tibble(bundle$entry)

import json
import urllib.request
from pathlib import Path
import pandas as pd

URL = "https://disease.sh/v3/covid-19/countries?allowNull=false"
CACHE = Path("api_cache.csv")   # truly local file in the current working directory

# Some APIs reject urllib's default User-Agent ("Python-urllib/...") with HTTP 403.
request = urllib.request.Request(URL, headers={"User-Agent": "data-science-course"})

try:
    with urllib.request.urlopen(request, timeout=5) as response:
        df_api = pd.DataFrame(json.load(response))
    df_api.to_csv(CACHE, index=False)         # keep a copy for offline re-runs
    print(f"API call succeeded: {len(df_api)} countries")
except Exception as exc:
    if CACHE.exists():
        df_api = pd.read_csv(CACHE)            # genuine local fallback (no network)
        print(f"API unreachable ({exc}). Using local cache {CACHE}.")
    else:
        raise SystemExit(f"API unreachable and no local cache {CACHE}: {exc}")

library(httr2)
library(jsonlite)

url   <- "https://disease.sh/v3/covid-19/countries?allowNull=false"
cache <- "api_cache.csv"   # truly local file in the current working directory

df_api <- tryCatch({
  response <- request(url) |> req_timeout(5) |> req_perform()
  # flatten = TRUE unpacks the nested countryInfo field, so write_csv() can save it.
  df <- fromJSON(resp_body_string(response), flatten = TRUE) |> tibble::as_tibble()
  readr::write_csv(df, cache)                 # keep a copy for offline re-runs
  df
}, error = function(e) {
  if (file.exists(cache)) {
    message("API unreachable: ", e$message, ". Using local cache ", cache, ".")
    readr::read_csv(cache, show_col_types = FALSE)   # genuine local fallback
  } else {
    stop("API unreachable and no local cache ", cache, ": ", e$message)
  }
})

GET [Server]/Patient?birthdate=ge1950-01-01&_count=100
→ JSON-Bundle mit einer Liste von Patient-Ressourcen

import pandas as pd
import requests   # pinned in requirements.txt (otherwise: uv pip install requests)

# REDCap API (conceptual – substitute your own token and URL).
payload = {
    "token": "MY_API_TOKEN",     # never store in code – use environment variable
    "content": "record",
    "format": "json",
    "type": "flat",
}
r = requests.post("https://redcap.example.org/api/", data=payload, timeout=30)
df_redcap = pd.DataFrame(r.json())