上線之後：漂移、效能、偏誤與隱私

絕對率差 = .68000 − .56183 ≈ .11817
相對差 = (.68000 − .56183) / .56183 ≈ 21.03%

bin 0: [0, .25)
bin 1: [.25, .5)
bin 2: [.5, .75)
bin 3: [.75, 1]

TV = 0.5 × Σ_bins |p_bin − q_bin|
alert = TV > 0.2

TPR = TP / (TP + FN)  # 真實正類中，被判正的比例
FPR = FP / (FP + TN)  # 真實負類中，被判正的比例

python3 monitor_batches.py > stdout.json

"""Original constructed monitoring cases; not a calibrated detector or fairness/privacy guarantee."""
import json


def make_batch(scores=(.2, .4, .6, .8), change_b_labels=False, labels_available=True):
    rows = []
    for group in ("A", "B"):
        for _ in range(5):
            for i, score in enumerate(scores):
                label = int(i >= 2)
                if change_b_labels and group == "B":
                    label = 0
                rows.append({"group": group, "score": score,
                             "label": label if labels_available else None})
    return rows


def distribution(rows):
    # Fixed, left-closed bins [0,.25), [.25,.5), [.5,.75), [.75,1].
    counts = [0, 0, 0, 0]
    for row in rows:
        score = row["score"]
        if not 0 <= score <= 1:
            raise ValueError("Score outside [0,1]")
        counts[min(int(score * 4), 3)] += 1
    return {"counts": counts, "proportions": [n / len(rows) for n in counts]}


def labeled_metrics(rows):
    known = [r for r in rows if r["label"] is not None]
    if not known:
        return {"total": len(rows), "labeled": 0, "accuracy": None,
                "confusion": None, "TPR": None, "FPR": None}
    tp = sum(r["label"] == 1 and r["score"] >= .5 for r in known)
    fp = sum(r["label"] == 0 and r["score"] >= .5 for r in known)
    tn = sum(r["label"] == 0 and r["score"] < .5 for r in known)
    fn = sum(r["label"] == 1 and r["score"] < .5 for r in known)
    return {"total": len(rows), "labeled": len(known), "accuracy": (tp + tn) / len(known),
            "confusion": {"TP": tp, "FP": fp, "TN": tn, "FN": fn},
            "TPR": tp / (tp + fn) if tp + fn else None,
            "FPR": fp / (fp + tn) if fp + tn else None}


def describe(rows, reference_distribution):
    actual = distribution(rows)
    tv = .5 * sum(abs(a - b) for a, b in zip(actual["proportions"], reference_distribution["proportions"]))
    return {"score_distribution": actual, "score_histogram_total_variation": tv,
            "handset_threshold": .2, "histogram_alert": tv > .2,
            "all": labeled_metrics(rows),
            "groups": {g: labeled_metrics([r for r in rows if r["group"] == g]) for g in ("A", "B")}}


def main():
    reference = make_batch()
    reference_distribution = distribution(reference)
    cases = {"reference": describe(reference, reference_distribution),
        "same_scores_changed_B_labels": describe(make_batch(change_b_labels=True), reference_distribution),
        "shifted_scores_same_correct_decisions": describe(make_batch(scores=(.05, .1, .9, .95)), reference_distribution),
        "same_scores_labels_delayed": describe(make_batch(change_b_labels=True, labels_available=False), reference_distribution)}
    assert cases["reference"]["all"]["accuracy"] == 1
    changed = cases["same_scores_changed_B_labels"]
    assert changed["score_histogram_total_variation"] == 0 and not changed["histogram_alert"]
    assert changed["all"]["accuracy"] == .75
    assert changed["groups"]["A"]["accuracy"] == 1 and changed["groups"]["B"]["accuracy"] == .5
    assert changed["groups"]["B"]["TPR"] is None and changed["groups"]["B"]["FPR"] == .5
    shifted = cases["shifted_scores_same_correct_decisions"]
    assert shifted["score_histogram_total_variation"] == .5 and shifted["histogram_alert"]
    assert shifted["all"]["accuracy"] == 1
    assert cases["same_scores_labels_delayed"]["all"]["accuracy"] is None
    print(json.dumps({"scope": "Four constructed 40-row batches, fixed score threshold .5, no model training or live monitor",
        "cases": cases,
        "limits": ["Histogram drift is score-distribution drift only, not all raw features or all concept changes.",
            "Threshold .2 is hand chosen, not calibrated for sampling noise, false alarms or multiple tests.",
            "Labels are changed by construction; real shifts require investigation and trustworthy feedback.",
            "A/B are synthetic cohorts, not actual protected demographics or proof of fairness/unfairness.",
            "Undefined rates are null, not zero; missing labels do not authorize reporting accuracy.",
            "Aggregate counts alone do not establish anonymity, privacy compliance or differential privacy.",
            "No latency/load, alert service, privacy audit or production outcome was measured."]}, indent=2, allow_nan=False))


if __name__ == "__main__":
    main()

DS71 image sources and modifications

Figures 1–2 and cover: TensorFlow documentation authors, Fairness Indicators (https://www.tensorflow.org/tfx/guide/fairness_indicators) and TensorFlow Model Analysis Visualizations (https://www.tensorflow.org/tfx/model_analysis/visualizations). The source pages license their documentation content under CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/), except where otherwise noted. These selected images are the pages' own documentation screenshots, not code samples. Main files are reproduced unchanged. The mobile view of Figure 1 preserves the whole screenshot first and repeats its complete right metric panel at larger size, using source pixels only. The cover fits the whole Figure 2 chart proportionally on a white canvas. No data, labels or plot marks are added or changed. The height slices and the three-point AUC graph are documentation examples, not this project's synthetic batch results, a score-drift detector, or proof of fairness.

Figure 3: Microsoft Corporation and Fairlearn contributors, Credit Loan Decisions, Fairlearn 0.12.0, https://fairlearn.org/v0.12/auto_examples/plot_credit_loan_decisions.html . Pinned source: https://github.com/fairlearn/fairlearn/blob/aeac30d6b1adcff7d24e6af69990ca1552b81210/examples/plot_credit_loan_decisions.py . Figure 008 is the ExponentiatedGradient selected inner-model group-metrics chart (epsilon=.01, index=14), in the section Selecting a suitable inner model, after the earlier separate ThresholdOptimizer section. It is not ThresholdOptimizer output. The original main PNG is unchanged; the mobile version preserves the complete original first and then repeats the three original metric panels in order at larger size. Only source-pixel crops and proportional resizing/white spacing are used, without relabeling or changing plotted data.

Data credit for Figure 3: I-Cheng Yeh (2009), Default of Credit Card Clients, UCI Machine Learning Repository, https://doi.org/10.24432/C55S3H and https://archive.ics.uci.edu/dataset/350/default+of+credit+card+clients , CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/). The Fairlearn example drops/renames fields and adds a semi-synthetic Interest feature generated from Y and SEX to construct a teaching disparity. It also uses test-set metrics when selecting the inner model; the displayed metrics are not untouched-holdout evidence, this project's results, a representative population finding, or a production lending recommendation. No original dataset records are redistributed in this article.

Fairlearn original MIT License (verbatim)
    MIT License

    Copyright (c) Microsoft Corporation and Fairlearn contributors.

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all
    copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    SOFTWARE