# core


<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->

## Setup & data

We use the [Northwind
dataset](https://github.com/neo4j-contrib/northwind-neo4j) — a classic
sample database with customers, orders, and employees tables that share
keys across them.

``` python
import httpx
from pathlib import Path

base = "https://raw.githubusercontent.com/neo4j-contrib/northwind-neo4j/refs/heads/master/data"
files = ["customers.csv", "orders.csv", "employees.csv"]
dest = Path("../data")
dest.mkdir(exist_ok=True)

for f in files:
    (dest/f).write_bytes(httpx.get(f"{base}/{f}").content)
    print(f"✓ {f}")
```

    ✓ customers.csv
    ✓ orders.csv
    ✓ employees.csv

``` python
import pandas as pd
for f in files:
    df = pd.read_csv(dest/f, on_bad_lines='skip')
    print(f"\n{f}: {df.shape}")
    print(df.columns.tolist())
    display(df.head(2))
```


    customers.csv: (67, 11)
    ['customerID', 'companyName', 'contactName', 'contactTitle', 'address', 'city', 'region', 'postalCode', 'country', 'phone', 'fax']

<div class="prose">
&#10;<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }
&#10;    .dataframe tbody tr th {
        vertical-align: top;
    }
&#10;    .dataframe thead th {
        text-align: right;
    }
</style>

<table class="dataframe" data-quarto-postprocess="true" data-border="1">
<thead>
<tr style="text-align: right;">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">customerID</th>
<th data-quarto-table-cell-role="th">companyName</th>
<th data-quarto-table-cell-role="th">contactName</th>
<th data-quarto-table-cell-role="th">contactTitle</th>
<th data-quarto-table-cell-role="th">address</th>
<th data-quarto-table-cell-role="th">city</th>
<th data-quarto-table-cell-role="th">region</th>
<th data-quarto-table-cell-role="th">postalCode</th>
<th data-quarto-table-cell-role="th">country</th>
<th data-quarto-table-cell-role="th">phone</th>
<th data-quarto-table-cell-role="th">fax</th>
</tr>
</thead>
<tbody>
<tr>
<td data-quarto-table-cell-role="th">0</td>
<td>ALFKI</td>
<td>Alfreds Futterkiste</td>
<td>Maria Anders</td>
<td>Sales Representative</td>
<td>Obere Str. 57</td>
<td>Berlin</td>
<td>NaN</td>
<td>12209</td>
<td>Germany</td>
<td>030-0074321</td>
<td>030-0076545</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">1</td>
<td>ANATR</td>
<td>Ana Trujillo Emparedados y helados</td>
<td>Ana Trujillo</td>
<td>Owner</td>
<td>Avda. de la Constitución 2222</td>
<td>México D.F.</td>
<td>NaN</td>
<td>05021</td>
<td>Mexico</td>
<td>(5) 555-4729</td>
<td>(5) 555-3745</td>
</tr>
</tbody>
</table>

</div>
&#10;</div>


    orders.csv: (654, 14)
    ['orderID', 'customerID', 'employeeID', 'orderDate', 'requiredDate', 'shippedDate', 'shipVia', 'freight', 'shipName', 'shipAddress', 'shipCity', 'shipRegion', 'shipPostalCode', 'shipCountry']

<div class="prose">
&#10;<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }
&#10;    .dataframe tbody tr th {
        vertical-align: top;
    }
&#10;    .dataframe thead th {
        text-align: right;
    }
</style>

<table class="dataframe" data-quarto-postprocess="true" data-border="1">
<thead>
<tr style="text-align: right;">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">orderID</th>
<th data-quarto-table-cell-role="th">customerID</th>
<th data-quarto-table-cell-role="th">employeeID</th>
<th data-quarto-table-cell-role="th">orderDate</th>
<th data-quarto-table-cell-role="th">requiredDate</th>
<th data-quarto-table-cell-role="th">shippedDate</th>
<th data-quarto-table-cell-role="th">shipVia</th>
<th data-quarto-table-cell-role="th">freight</th>
<th data-quarto-table-cell-role="th">shipName</th>
<th data-quarto-table-cell-role="th">shipAddress</th>
<th data-quarto-table-cell-role="th">shipCity</th>
<th data-quarto-table-cell-role="th">shipRegion</th>
<th data-quarto-table-cell-role="th">shipPostalCode</th>
<th data-quarto-table-cell-role="th">shipCountry</th>
</tr>
</thead>
<tbody>
<tr>
<td data-quarto-table-cell-role="th">0</td>
<td>10248</td>
<td>VINET</td>
<td>5</td>
<td>1996-07-04 00:00:00.000</td>
<td>1996-08-01 00:00:00.000</td>
<td>1996-07-16 00:00:00.000</td>
<td>3</td>
<td>32.38</td>
<td>Vins et alcools Chevalier</td>
<td>59 rue de l'Abbaye</td>
<td>Reims</td>
<td>NaN</td>
<td>51100</td>
<td>France</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">1</td>
<td>10249</td>
<td>TOMSP</td>
<td>6</td>
<td>1996-07-05 00:00:00.000</td>
<td>1996-08-16 00:00:00.000</td>
<td>1996-07-10 00:00:00.000</td>
<td>1</td>
<td>11.61</td>
<td>Toms Spezialitäten</td>
<td>Luisenstr. 48</td>
<td>Münster</td>
<td>NaN</td>
<td>44087</td>
<td>Germany</td>
</tr>
</tbody>
</table>

</div>
&#10;</div>


    employees.csv: (9, 18)
    ['employeeID', 'lastName', 'firstName', 'title', 'titleOfCourtesy', 'birthDate', 'hireDate', 'address', 'city', 'region', 'postalCode', 'country', 'homePhone', 'extension', 'photo', 'notes', 'reportsTo', 'photoPath']

<div class="prose">
&#10;<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }
&#10;    .dataframe tbody tr th {
        vertical-align: top;
    }
&#10;    .dataframe thead th {
        text-align: right;
    }
</style>

<table class="dataframe" data-quarto-postprocess="true" data-border="1">
<thead>
<tr style="text-align: right;">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">employeeID</th>
<th data-quarto-table-cell-role="th">lastName</th>
<th data-quarto-table-cell-role="th">firstName</th>
<th data-quarto-table-cell-role="th">title</th>
<th data-quarto-table-cell-role="th">titleOfCourtesy</th>
<th data-quarto-table-cell-role="th">birthDate</th>
<th data-quarto-table-cell-role="th">hireDate</th>
<th data-quarto-table-cell-role="th">address</th>
<th data-quarto-table-cell-role="th">city</th>
<th data-quarto-table-cell-role="th">region</th>
<th data-quarto-table-cell-role="th">postalCode</th>
<th data-quarto-table-cell-role="th">country</th>
<th data-quarto-table-cell-role="th">homePhone</th>
<th data-quarto-table-cell-role="th">extension</th>
<th data-quarto-table-cell-role="th">photo</th>
<th data-quarto-table-cell-role="th">notes</th>
<th data-quarto-table-cell-role="th">reportsTo</th>
<th data-quarto-table-cell-role="th">photoPath</th>
</tr>
</thead>
<tbody>
<tr>
<td data-quarto-table-cell-role="th">0</td>
<td>1</td>
<td>Davolio</td>
<td>Nancy</td>
<td>Sales Representative</td>
<td>Ms.</td>
<td>1948-12-08 00:00:00.000</td>
<td>1992-05-01 00:00:00.000</td>
<td>507 - 20th Ave. E. Apt. 2A</td>
<td>Seattle</td>
<td>WA</td>
<td>98122</td>
<td>USA</td>
<td>(206) 555-9857</td>
<td>5467</td>
<td>0x151C2F00020000000D000E0014002100FFFFFFFF4269...</td>
<td>Education includes a BA in psychology from Col...</td>
<td>2.0</td>
<td>http://accweb/emmployees/davolio.bmp</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">1</td>
<td>2</td>
<td>Fuller</td>
<td>Andrew</td>
<td>Vice President, Sales</td>
<td>Dr.</td>
<td>1952-02-19 00:00:00.000</td>
<td>1992-08-14 00:00:00.000</td>
<td>908 W. Capital Way</td>
<td>Tacoma</td>
<td>WA</td>
<td>98401</td>
<td>USA</td>
<td>(206) 555-9482</td>
<td>3457</td>
<td>0x151C2F00020000000D000E0014002100FFFFFFFF4269...</td>
<td>Andrew received his BTS commercial in 1974 and...</td>
<td>NaN</td>
<td>http://accweb/emmployees/fuller.bmp</td>
</tr>
</tbody>
</table>

</div>
&#10;</div>

## Table configuration

Each table is configured with **key groups** (`id1`, `id2`, …) and a
list of columns to **mask**. Key groups ensure consistent
pseudonymisation across tables — e.g. `customerID` maps to the same UUID
in both `customers` and `orders` because both use group `id1`.

``` python
tables = {
    'customers': {
        'id1': 'customerID',
        'mask': ['contactName', 'address', 'phone', 'companyName', 'fax', 'city']
    },
    'orders': {
        'id1': 'customerID',
        'id2': 'employeeID',
        'mask': ['shipName', 'shipAddress']
    },
    'employees': {
        'id2': 'employeeID',
        'mask': ['firstName', 'lastName', 'address', 'homePhone', 'birthDate', 'notes']
    },
}
```

## Design decisions

### Why UUID4 for key pseudonymisation?

UUID4 is generated from **cryptographically random** bits, no timestamp,
MAC address, or sequence baked in. The fake ID reveals nothing about the
original. With 122 random bits, collision probability is astronomically
small.

### Shared vault for masked columns

Masked columns (names, addresses, etc.) use a **flat shared dict**
across all tables. If `"Maria Anders"` appears in both `customers` and
`orders`, it gets the same hex token. This preserves referential
consistency without leaking the original value.

### `setdefault` pattern

Both `key_maps` and `vault` use `dict.setdefault()` , a get-or-insert in
one call. This ensures existing mappings are reused and new values are
only generated for unseen originals:

``` python
km = self.key_maps.setdefault(group, {})  # create or retrieve inner dict
km.setdefault(val, str(uuid.uuid4()))     # only insert if missing
```

### Encryption at rest with Fernet

The mappings (`key_maps` + `vault`) are the most sensitive artefact ;
they’re the keys to de-anonymise everything. We use **Fernet**
(AES-128-CBC + HMAC) from the `cryptography` library. The encryption key
is never stored on the
[`CloakDF`](https://giordafrancis.github.io/cloakdf/core.html#cloakdf)
instance — it’s passed in at `save()`/`load()` time, keeping the class
stateless with respect to secrets.

## CloakDF class

------------------------------------------------------------------------

<a
href="https://github.com/giordafrancis/cloakdf/blob/master/cloakdf/core.py#L17"
target="_blank" style="float:right; font-size:smaller">source</a>

### CloakDF

``` python

def CloakDF(
    tables
):

```

*Pseudonymise and mask DataFrames consistently across related tables.*

## Round-trip test

Encode all three tables, save encrypted mappings, load from a fresh
instance, decode, and verify we get the originals back.

``` python
import os

# 1. Generate key and store in env var
key = CloakDF.generate_key()
os.environ['CLOAKDF_KEY'] = key.decode()
loaded_key = CloakDF.load_key(env_var='CLOAKDF_KEY')
assert loaded_key == key
print("✓ Key round-tripped via env var")

# 2. Encode all tables
clk = CloakDF(tables)
originals, encoded = {}, {}
for name in tables:
    df = pd.read_csv(dest/f"{name}.csv", on_bad_lines='skip')
    for k, v in tables[name].items():
        if k.startswith('id'): df[v] = df[v].astype(str)
    originals[name] = df
    encoded[name] = clk.encode(name, originals[name])
    print(f"✓ Encoded {name}")

# 3. Save, load with env key, decode and compare
# --- Key loading options (uncomment one) 
# loaded_key = CloakDF.load_key(path=dest/"secret.key")
# loaded_key = CloakDF.load_key(env_var='CLOAKDF_KEY')
clk.save(dest/"mappings.enc", loaded_key)
clk2 = CloakDF.load(dest/"mappings.enc", loaded_key, tables)
for name in tables:
    decoded = clk2.decode(name, encoded[name])
    pd.testing.assert_frame_equal(decoded, originals[name])
    print(f"✓ Decoded {name} matches original")
```

    ✓ Key round-tripped via env var
    ✓ Encoded customers
    ✓ Encoded orders
    ✓ Encoded employees
    ✓ Decoded customers matches original
    ✓ Decoded orders matches original
    ✓ Decoded employees matches original

``` python
cols = ['employeeID', 'firstName', 'lastName', 'title', 'address', 'homePhone']
display(encoded['employees'][cols].head(3))
display(decoded[cols].head(3))
```

<div class="prose">
&#10;<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }
&#10;    .dataframe tbody tr th {
        vertical-align: top;
    }
&#10;    .dataframe thead th {
        text-align: right;
    }
</style>

<table class="dataframe" data-quarto-postprocess="true" data-border="1">
<thead>
<tr style="text-align: right;">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">employeeID</th>
<th data-quarto-table-cell-role="th">firstName</th>
<th data-quarto-table-cell-role="th">lastName</th>
<th data-quarto-table-cell-role="th">title</th>
<th data-quarto-table-cell-role="th">address</th>
<th data-quarto-table-cell-role="th">homePhone</th>
</tr>
</thead>
<tbody>
<tr>
<td data-quarto-table-cell-role="th">0</td>
<td>d64fe22d-1661-4591-925d-444a16a2b05d</td>
<td>347dbd51390f</td>
<td>5b5ed8433ade</td>
<td>Sales Representative</td>
<td>e8973b766f93</td>
<td>cabbc198a844</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">1</td>
<td>5a2fb4ef-1517-4cd1-8f78-8b194a38a8d3</td>
<td>1d2f56ece163</td>
<td>f6369845f53e</td>
<td>Vice President, Sales</td>
<td>432fb00bda39</td>
<td>eb8a4817cb53</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">2</td>
<td>986a68a7-1539-4108-b132-76f69cded2a2</td>
<td>bc63cd2ad727</td>
<td>77df49f480ac</td>
<td>Sales Representative</td>
<td>2b9e99510ab1</td>
<td>d92bb310c745</td>
</tr>
</tbody>
</table>

</div>
&#10;</div>

<div class="prose">
&#10;<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }
&#10;    .dataframe tbody tr th {
        vertical-align: top;
    }
&#10;    .dataframe thead th {
        text-align: right;
    }
</style>

<table class="dataframe" data-quarto-postprocess="true" data-border="1">
<thead>
<tr style="text-align: right;">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">employeeID</th>
<th data-quarto-table-cell-role="th">firstName</th>
<th data-quarto-table-cell-role="th">lastName</th>
<th data-quarto-table-cell-role="th">title</th>
<th data-quarto-table-cell-role="th">address</th>
<th data-quarto-table-cell-role="th">homePhone</th>
</tr>
</thead>
<tbody>
<tr>
<td data-quarto-table-cell-role="th">0</td>
<td>1</td>
<td>Nancy</td>
<td>Davolio</td>
<td>Sales Representative</td>
<td>507 - 20th Ave. E. Apt. 2A</td>
<td>(206) 555-9857</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">1</td>
<td>2</td>
<td>Andrew</td>
<td>Fuller</td>
<td>Vice President, Sales</td>
<td>908 W. Capital Way</td>
<td>(206) 555-9482</td>
</tr>
<tr>
<td data-quarto-table-cell-role="th">2</td>
<td>3</td>
<td>Janet</td>
<td>Leverling</td>
<td>Sales Representative</td>
<td>722 Moss Bay Blvd.</td>
<td>(206) 555-3412</td>
</tr>
</tbody>
</table>

</div>
&#10;</div>