Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions examples/cleaning_memory_replay.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""Cleaning memory + replay via ``fd.learn_cleaning_memory`` (new in 1.1).

Record a reviewed ``CleanReport`` as reusable, server-free memory (JSON or
SQLite), then replay the accepted decisions on new batches of the *same*
dataset. Replay only happens when the new data still looks like what the
memory learned (column set + coarse dtypes); if it drifted too far, replay is
**blocked and explained** on the report rather than applied blindly. Run:

python examples/cleaning_memory_replay.py
"""

import pandas as pd

import freshdata as fd


def main() -> None:
# Day 1: a human reviews and accepts freshdata's cleaning decisions.
day1 = pd.DataFrame(
{
"Customer Name": [" Alice ", "Bob", "N/A", "Alice"],
"Signup Date": ["2024-01-05", "2024-02-30", "2024/03/01", "2024-01-05"],
"Revenue": ["1200", "980", "-", "1200"],
}
)
cleaned1, report1 = fd.clean(day1, return_report=True)

memory = fd.learn_cleaning_memory(day1, decisions=report1, dataset_id="crm_daily")
memory.to_json("crm_daily_memory.json")
print(memory.summary())

# Day 2: same shape/dtypes -> memory matches, decisions replay automatically.
day2 = pd.DataFrame(
{
"Customer Name": [" Dave ", "Erin", "N/A"],
"Signup Date": ["2024-04-01", "2024-04-02", None],
"Revenue": ["2100", "-", "1750"],
}
)
loaded = fd.load_cleaning_memory("crm_daily_memory.json")
cleaned2, report2 = fd.clean(day2, memory=loaded, return_report=True)

replayed = [a for a in report2.actions if a.memory_influenced]
print(f"day 2: {len(replayed)} decision(s) replayed from memory")
for w in report2.warnings:
print("warning:", w)

# Day 3: the feed drifted (a learned column is gone) -> replay is blocked,
# not silently skipped — the report says exactly why.
day3 = pd.DataFrame({"Revenue": ["500", "600"]})
cleaned3, report3 = fd.clean(day3, memory=loaded, return_report=True)
print("day 3 warnings:", report3.warnings)
print("day 3 recommendations:", report3.recommendations)

# Inspect what changed between two learned memories (e.g. after a re-review).
day2_review_report = fd.clean(day2, return_report=True)[1]
memory2 = fd.learn_cleaning_memory(day2, decisions=day2_review_report, dataset_id="crm_daily")
print(memory.diff(memory2))


if __name__ == "__main__":
main()
63 changes: 63 additions & 0 deletions examples/interactive_html_reports.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""Interactive HTML reports via ``freshdata.render`` (new in 1.1).

``CleanReport``, ``Profile``, ``CleanPlan`` and ``ExplainReport`` all gain
``to_html()`` / ``_repr_html_()`` / ``.show()``: a collapsible action timeline
and filterable audit ledger, an inline quality cockpit, decision cards for a
proposed plan, and a before/after diff explorer. Everything is self-contained
HTML (scoped CSS + a little vanilla JS) — no optional deps required. Installing
``freshdata-cleaner[viz]`` / ``[notebook]`` only upgrades the output (itables,
plotly, great-tables, anywidget). Run:

python examples/interactive_html_reports.py
"""

import pandas as pd

import freshdata as fd


def main() -> None:
df = pd.DataFrame(
{
"Customer Name": [" Alice ", "Bob", "N/A", "Alice", " Carol"],
"Signup Date": ["2024-01-05", "2024-02-30", "2024/03/01", "2024-01-05", None],
"Revenue": ["1200", "980", "-", "1200", "15000"],
}
)

# 1. CleanReport — action timeline + audit ledger.
cleaned, report = fd.clean(df, return_report=True)
report_path = report.show() # opens inline in Jupyter, else writes an .html file
print(f"CleanReport -> {report_path}")

# 2. Profile — inline quality cockpit for the raw data.
prof = fd.profile(df)
profile_path = prof.show()
print(f"Profile -> {profile_path}")

# 3. CleanPlan — decision cards / strategy diff grid for a *proposed*, not-yet-
# applied plan, so a reviewer can approve/reject before anything runs.
plan = fd.suggest_plan(df)
plan_path = plan.show()
print(f"CleanPlan -> {plan_path}")

# 4. ExplainReport — before/after diff explorer for the whole run (per-column
# dtype/changed-cell stats via .to_frame(), narratives, warnings).
explain = fd.explain_clean(df)
explain_path = explain.show()
print(explain.summary())
print(f"ExplainReport -> {explain_path}")

# 5. compare_plans / compare_clean / infer_roles — return a ``ReportFrame``
# (a DataFrame subclass): behaves like a normal frame but also renders
# as HTML via .to_html()/.show().
strategy_diff = fd.compare_plans(df, strategies=("conservative", "balanced", "aggressive"))
print(strategy_diff)
strategy_diff.show()

roles = fd.infer_roles(df)
print(roles)


if __name__ == "__main__":
main()
Loading