| |
| |
|
|
| """ |
| Parse *_NB_single_dataset.txt reports and emit LaTeX tables. |
| |
| Now supports: |
| • Full "everything" longtable (as before) |
| • Option B: TWO COMPACT portrait tables (RMSE table + MAE table) |
| |
| Examples |
| -------- |
| # Make compact + full tables for all reports in a dir |
| python3 make_nb_tables_from_reports.py --in ./plotting --out ./plotting/tex |
| |
| # Only compact tables |
| python3 make_nb_tables_from_reports.py --in ./plotting --out ./plotting/tex --compact_only |
| |
| # Specific files |
| python3 make_nb_tables_from_reports.py \ |
| --files ./plotting/qm9_NB_single_dataset.txt ./plotting/boilingpoint_NB_single_dataset.txt \ |
| --out ./plotting/tex --compact_only |
| """ |
|
|
| import argparse |
| import re |
| from pathlib import Path |
| from typing import Dict, List, Tuple |
| import pandas as pd |
|
|
| NUM = r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?" |
|
|
|
|
| def esc_tex(s: str) -> str: |
| return s.replace("_", r"\_") |
|
|
|
|
| def parse_header(lines: List[str]) -> Dict[str, str]: |
| out = {} |
| for ln in lines[:60]: |
| m = re.search(r"^Dataset:\s*(.+?)\s+—", ln) |
| if m: |
| out["dataset"] = m.group(1).strip() |
| m = re.search(r"^Control exp_id:\s*(.+)$", ln) |
| if m: |
| out["control"] = m.group(1).strip() |
| m = re.search(r"^k folds:\s*(\d+),\s*alpha:\s*(" + NUM + r")", ln) |
| if m: |
| out["k"] = int(m.group(1)) |
| out["alpha"] = float(m.group(2)) |
| return out |
|
|
|
|
| def find_block(lines: List[str], start_pat: str) -> Tuple[int, int]: |
| start = -1 |
| pat = re.compile(start_pat) |
| for i, ln in enumerate(lines): |
| if pat.search(ln): |
| start = i |
| break |
| if start < 0: |
| return (-1, -1) |
| end = len(lines) |
| for j in range(start + 1, len(lines)): |
| if not lines[j].strip(): |
| end = j |
| break |
| return (start, end) |
|
|
|
|
| def parse_nb_block(lines: List[str]) -> pd.DataFrame: |
| hdr_idx = None |
| for i, ln in enumerate(lines): |
| if re.search(r"^\s*comparison\s+", ln) and "mean_diff_RMSE" in ln: |
| hdr_idx = i |
| break |
| if hdr_idx is None: |
| return pd.DataFrame() |
|
|
| data_lines = [] |
| for ln in lines[hdr_idx + 1 :]: |
| s = ln.rstrip() |
| if not s: |
| break |
| if set(s) <= set("- "): |
| continue |
| data_lines.append(s) |
|
|
| rows = [] |
| for ln in data_lines: |
| |
| m = re.match(r"^\s*(.+?)\s+(" + NUM + r"(?:\s+" + NUM + r"){9})\s*$", ln) |
| if not m: |
| parts = re.split(r"\s{2,}", ln.strip()) |
| if len(parts) < 11: |
| continue |
| comp, *nums = parts |
| else: |
| comp = m.group(1).strip() |
| nums = re.split(r"\s+", m.group(2).strip()) |
| if len(nums) != 10: |
| continue |
| vals = list(map(float, nums)) |
| rows.append( |
| { |
| "comparison": comp, |
| "mean_diff_RMSE": vals[0], |
| "t_NB_RMSE": vals[1], |
| "p_raw_RMSE": vals[2], |
| "mean_diff_MAE": vals[3], |
| "t_NB_MAE": vals[4], |
| "p_raw_MAE": vals[5], |
| "CI_RMSE_low": vals[6], |
| "CI_RMSE_high": vals[7], |
| "CI_MAE_low": vals[8], |
| "CI_MAE_high": vals[9], |
| } |
| ) |
| return pd.DataFrame(rows) |
|
|
|
|
| def parse_holm_block(lines: List[str], metric_label: str) -> pd.DataFrame: |
| hdr = None |
| for i, ln in enumerate(lines): |
| if re.search(r"^\s*comparison\s+", ln) and "p_raw" in ln and "p_holm" in ln: |
| hdr = i |
| break |
| if hdr is None: |
| return pd.DataFrame() |
|
|
| data_lines = [] |
| for ln in lines[hdr + 1 :]: |
| s = ln.rstrip() |
| if not s: |
| break |
| if set(s) <= set("- "): |
| continue |
| data_lines.append(s) |
|
|
| rows = [] |
| for ln in data_lines: |
| m = re.match(r"^\s*(.+?)\s+(" + NUM + r")\s+(" + NUM + r")\s+(\w+)\s*$", ln) |
| if m: |
| comp = m.group(1).strip() |
| p_raw = float(m.group(2)) |
| p_holm = float(m.group(3)) |
| sig = m.group(4).strip().lower().startswith("t") |
| else: |
| parts = re.split(r"\s{2,}", ln.strip()) |
| if len(parts) < 4: |
| continue |
| comp = parts[0].strip() |
| p_holm = float(parts[2]) |
| sig = parts[3].strip().lower().startswith("t") |
| rows.append( |
| { |
| "comparison": comp, |
| f"p_holm_{metric_label}": p_holm, |
| f"sig_{metric_label}": sig, |
| } |
| ) |
| return pd.DataFrame(rows) |
|
|
|
|
| |
|
|
|
|
| def write_full_longtable(meta: Dict[str, str], df: pd.DataFrame, outpath: Path): |
| dataset = meta.get("dataset", "dataset") |
| control = meta.get("control", "control") |
| k = int(meta.get("k", 5)) |
| alpha = float(meta.get("alpha", 0.05)) |
|
|
| header = rf""" |
| % Auto-generated from NB_single_dataset report |
| % Requires: \usepackage{{booktabs,longtable,siunitx}} |
| \begin{{longtable}}{{@{{}}l |
| S S S S S S S |
| S |
| S S S S S S S |
| @{{}}}} |
| \caption{{Full NB-corrected one-sided tests (outer folds, $K={k}$, $\alpha={alpha}$) comparing control \texttt{{{esc_tex(control)}}} against each competitor on dataset \texttt{{{esc_tex(dataset)}}}. |
| Positive $\Delta$ (competitor $-$ control) favors the control (lower loss). |
| One-sided alternative $\mathbb{{E}}[\bar d^{{(L)}}] > 0$. |
| Holm step-down controls FWER per metric family (RMSE and MAE reported separately). |
| CIs are NB-style two-sided.}} |
| \label{{tab:{esc_tex(dataset)}_nb_full}}\\ |
| \toprule |
| & \multicolumn{{7}}{{c}}{{\textbf{{RMSE family}}}} & & |
| \multicolumn{{7}}{{c}}{{\textbf{{MAE family}}}}\\ |
| \cmidrule(lr){{2-8}}\cmidrule(lr){{10-16}} |
| Comparison |
| & {{\(\Delta\)RMSE}} |
| & {{$t_{{\text{{NB}}}}$}} |
| & {{p (raw)}} |
| & {{p\(_{{\text{{Holm}}}}\)}} |
| & {{\(\text{{Sig}}\)}} |
| & {{CI\(_{{\text{{low}}}}\)}} |
| & {{CI\(_{{\text{{high}}}}\)}} |
| & {{}} % spacer |
| & {{\(\Delta\)MAE}} |
| & {{$t_{{\text{{NB}}}}$}} |
| & {{p (raw)}} |
| & {{p\(_{{\text{{Holm}}}}\)}} |
| & {{\(\text{{Sig}}\)}} |
| & {{CI\(_{{\text{{low}}}}\)}} |
| & {{CI\(_{{\text{{high}}}}\)}}\\ |
| \midrule |
| \endfirsthead |
| \toprule |
| & \multicolumn{{7}}{{c}}{{\textbf{{RMSE family}}}} & & |
| \multicolumn{{7}}{{c}}{{\textbf{{MAE family}}}}\\ |
| \cmidrule(lr){{2-8}}\cmidrule(lr){{10-16}} |
| Comparison |
| & {{\(\Delta\)RMSE}} |
| & {{$t_{{\text{{NB}}}}$}} |
| & {{p (raw)}} |
| & {{p\(_{{\text{{Holm}}}}\)}} |
| & {{\(\text{{Sig}}\)}} |
| & {{CI\(_{{\text{{low}}}}\)}} |
| & {{CI\(_{{\text{{high}}}}\)}} |
| & {{}} % spacer |
| & {{\(\Delta\)MAE}} |
| & {{$t_{{\text{{NB}}}}$}} |
| & {{p (raw)}} |
| & {{p\(_{{\text{{Holm}}}}\)}} |
| & {{\(\text{{Sig}}\)}} |
| & {{CI\(_{{\text{{low}}}}\)}} |
| & {{CI\(_{{\text{{high}}}}\)}}\\ |
| \midrule |
| \endhead |
| \midrule |
| \multicolumn{{16}}{{r}}{{\emph{{Continued on next page}}}}\\ |
| \midrule |
| \endfoot |
| \bottomrule |
| \endlastfoot |
| """ |
| lines = [] |
| for _, r in df.iterrows(): |
| lines.append( |
| f"{esc_tex(r['comparison'])} & " |
| f"{r['mean_diff_RMSE']:.6f} & {r['t_NB_RMSE']:.6f} & {r['p_raw_RMSE']:.6f} & {r['p_holm_RMSE']:.6f} & {1 if r['sig_RMSE'] else 0} & {r['CI_RMSE_low']:.6f} & {r['CI_RMSE_high']:.6f} & & " |
| f"{r['mean_diff_MAE']:.6f} & {r['t_NB_MAE']:.6f} & {r['p_raw_MAE']:.6f} & {r['p_holm_MAE']:.6f} & {1 if r['sig_MAE'] else 0} & {r['CI_MAE_low']:.6f} & {r['CI_MAE_high']:.6f} \\\\" |
| ) |
| outpath.write_text( |
| header + "\n".join(lines) + "\n\\end{longtable}\n", encoding="utf-8" |
| ) |
|
|
|
|
| |
|
|
|
|
| def write_compact_tables(meta: Dict[str, str], df: pd.DataFrame, outdir: Path): |
| """ |
| Produce two small portrait tables: |
| • <dataset>_nb_compact_rmse.tex |
| • <dataset>_nb_compact_mae.tex |
| |
| Columns (per metric): Comparison, Δ, t_NB, CI_low, CI_high, p_Holm |
| """ |
| dataset = meta.get("dataset", "dataset") |
| control = meta.get("control", "control") |
| k = int(meta.get("k", 5)) |
|
|
| common_preamble = r""" |
| % Auto-generated compact NB tables |
| % Requires in preamble: \usepackage{booktabs,tabularx,siunitx} |
| % \sisetup{round-mode=places,round-precision=3,scientific-notation=true} |
| """ |
|
|
| |
| rmse_header = rf"""{common_preamble} |
| \begin{{table}}[t] |
| \centering |
| \small |
| \setlength{{\tabcolsep}}{{4pt}} |
| \caption{{RMSE: NB-corrected one-sided tests (outer folds, $K={k}$) on dataset \texttt{{{esc_tex(dataset)}}}; control \texttt{{{esc_tex(control)}}}. |
| Positive $\Delta$ (competitor $-$ control) favors control. Holm controls FWER.}} |
| \label{{tab:{esc_tex(dataset)}_nb_compact_rmse}} |
| \begin{{tabularx}}{{\linewidth}}{{@{{}}l S S S S S@{{}}}} |
| \toprule |
| Comparison & {{\(\Delta\)RMSE}} & {{$t_{{\text{{NB}}}}$}} & {{CI\(_{{\text{{low}}}}\)}} & {{CI\(_{{\text{{high}}}}\)}} & {{p\(_{{\text{{Holm}}}}\)}}\\ |
| \midrule |
| """ |
| rmse_rows = [] |
| for _, r in df.sort_values("p_holm_RMSE").iterrows(): |
| rmse_rows.append( |
| f"{esc_tex(r['comparison'])} & " |
| f"{r['mean_diff_RMSE']:.3f} & {r['t_NB_RMSE']:.3f} & " |
| f"{r['CI_RMSE_low']:.3f} & {r['CI_RMSE_high']:.3f} & {r['p_holm_RMSE']:.3g} \\\\" |
| ) |
| rmse_tex = ( |
| rmse_header |
| + "\n".join(rmse_rows) |
| + "\n\\bottomrule\n\\end{tabularx}\n\\end{table}\n" |
| ) |
| (outdir / f"{dataset}_nb_compact_rmse.tex").write_text(rmse_tex, encoding="utf-8") |
|
|
| |
| mae_header = rf"""{common_preamble} |
| \begin{{table}}[t] |
| \centering |
| \small |
| \setlength{{\tabcolsep}}{{4pt}} |
| \caption{{MAE: NB-corrected one-sided tests (outer folds, $K={k}$) on dataset \texttt{{{esc_tex(dataset)}}}; control \texttt{{{esc_tex(control)}}}. |
| Positive $\Delta$ (competitor $-$ control) favors control. Holm controls FWER.}} |
| \label{{tab:{esc_tex(dataset)}_nb_compact_mae}} |
| \begin{{tabularx}}{{\linewidth}}{{@{{}}l S S S S S@{{}}}} |
| \toprule |
| Comparison & {{\(\Delta\)MAE}} & {{$t_{{\text{{NB}}}}$}} & {{CI\(_{{\text{{low}}}}\)}} & {{CI\(_{{\text{{high}}}}\)}} & {{p\(_{{\text{{Holm}}}}\)}}\\ |
| \midrule |
| """ |
| mae_rows = [] |
| for _, r in df.sort_values("p_holm_MAE").iterrows(): |
| mae_rows.append( |
| f"{esc_tex(r['comparison'])} & " |
| f"{r['mean_diff_MAE']:.3f} & {r['t_NB_MAE']:.3f} & " |
| f"{r['CI_MAE_low']:.3f} & {r['CI_MAE_high']:.3f} & {r['p_holm_MAE']:.3g} \\\\" |
| ) |
| mae_tex = ( |
| mae_header |
| + "\n".join(mae_rows) |
| + "\n\\bottomrule\n\\end{tabularx}\n\\end{table}\n" |
| ) |
| (outdir / f"{dataset}_nb_compact_mae.tex").write_text(mae_tex, encoding="utf-8") |
|
|
|
|
| def process_file(path: Path, outdir: Path, make_full: bool, make_compact: bool): |
| txt = path.read_text(encoding="utf-8", errors="ignore").splitlines() |
| meta = parse_header(txt) |
|
|
| |
| nb_start, nb_end = find_block( |
| txt, r"^\s*--- NB-corrected t \(outer folds\) per competitor ---" |
| ) |
| if nb_start < 0: |
| raise RuntimeError(f"NB block not found in {path}") |
| df_nb = parse_nb_block(txt[nb_start:nb_end]) |
| if df_nb.empty: |
| raise RuntimeError(f"NB table parsing failed in {path}") |
|
|
| |
| hr_start, hr_end = find_block( |
| txt, r"^\s*--- Holm-adjusted p-values \(RMSE family\)\s*---" |
| ) |
| hm_start, hm_end = find_block( |
| txt, r"^\s*--- Holm-adjusted p-values \(MAE family\)\s*---" |
| ) |
| if hr_start < 0 or hm_start < 0: |
| raise RuntimeError(f"Holm blocks not found in {path}") |
| df_hr = parse_holm_block(txt[hr_start:hr_end], "RMSE") |
| df_hm = parse_holm_block(txt[hm_start:hm_end], "MAE") |
|
|
| |
| df = df_nb.merge(df_hr, on="comparison", how="left").merge( |
| df_hm, on="comparison", how="left" |
| ) |
|
|
| |
| dataset = meta.get("dataset", path.stem.replace("_NB_single_dataset", "")) |
| outdir.mkdir(parents=True, exist_ok=True) |
|
|
| if make_full: |
| full_path = outdir / f"{dataset}_nb_full_summary_from_report.tex" |
| |
| df_full = df.sort_values(["p_raw_RMSE", "p_raw_MAE"]).reset_index(drop=True) |
| write_full_longtable(meta, df_full, full_path) |
|
|
| if make_compact: |
| |
| write_compact_tables(meta, df, outdir) |
|
|
| print( |
| f"Processed: {path.name} -> {dataset} (full={make_full}, compact={make_compact})" |
| ) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument( |
| "--in", |
| dest="indir", |
| type=str, |
| default=None, |
| help="Directory containing *_NB_single_dataset.txt files.", |
| ) |
| ap.add_argument( |
| "--files", nargs="*", default=None, help="Explicit list of report files." |
| ) |
| ap.add_argument( |
| "--out", |
| dest="outdir", |
| type=str, |
| required=True, |
| help="Output directory for .tex tables.", |
| ) |
| ap.add_argument( |
| "--compact_only", |
| action="store_true", |
| help="Only write the compact RMSE/MAE tables (skip the full longtable).", |
| ) |
| args = ap.parse_args() |
|
|
| outdir = Path(args.outdir) |
| if args.files: |
| files = [Path(f) for f in args.files] |
| elif args.indir: |
| files = sorted(Path(args.indir).glob("*_NB_single_dataset.txt")) |
| else: |
| raise SystemExit("Provide either --in DIR or --files file1 file2 ...") |
|
|
| if not files: |
| raise SystemExit("No *_NB_single_dataset.txt files found.") |
|
|
| for f in files: |
| process_file(f, outdir, make_full=not args.compact_only, make_compact=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
