完成一個小型資料科學專案：從問題到驗收

logit = −3 + .025×distance + .16×queue
        − .7×promised_days + .7×fragile
p = sigmoid(logit)
late = 依固定 RNG、用機率 p 抽出的 0/1

train180 → fit StandardScaler → fit LogisticRegression
validation60 → 選固定候選閾值、選常數 baseline
凍結模型／閾值／baseline
最後 test60 → 計算預先宣告的驗收結果，不回頭改設定

假定錯誤成本 = 5 × FN + 1 × FP

python3 project_acceptance.py > stdout.json

python3 -m venv .venv
. .venv/bin/activate
python -m pip install numpy==1.26.3 scikit-learn==1.5.1
python project_acceptance.py > stdout.json

"""Original synthetic order-review project: predeclared split, validation choice, one test evaluation."""
import hashlib
import json
import math
import platform
import random

import numpy as np
import sklearn
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler


def metrics(labels, predicted):
    tp = sum(y == 1 and p == 1 for y, p in zip(labels, predicted))
    fp = sum(y == 0 and p == 1 for y, p in zip(labels, predicted))
    tn = sum(y == 0 and p == 0 for y, p in zip(labels, predicted))
    fn = sum(y == 1 and p == 0 for y, p in zip(labels, predicted))
    return {"n": len(labels), "TP": tp, "FP": fp, "TN": tn, "FN": fn,
        "cost_FN5_FP1": 5 * fn + fp,
        "recall": tp / (tp + fn) if tp + fn else None,
        "precision": tp / (tp + fp) if tp + fp else None,
        "review_count": tp + fp, "accuracy": (tp + tn) / len(labels)}


def main():
    # Declared before looking at the held-out test: synthetic teaching costs, not business estimates.
    thresholds = [.1, .2, .3, .4, .5, .6, .7, .8, .9]
    minimum_recall = .8
    rng = random.Random(42)
    records = []
    for row_id in range(300):
        distance = rng.uniform(5, 100)
        queue = rng.randrange(0, 21)
        promised_days = rng.choice([1, 2, 3, 4])
        fragile = rng.randrange(2)
        # All features represent information available at order creation, before the synthetic label.
        logit = -3 + .025 * distance + .16 * queue - .7 * promised_days + .7 * fragile
        label = int(rng.random() < 1 / (1 + math.exp(-logit)))
        records.append({"id": row_id, "features": [distance, queue, promised_days, fragile], "late": label})
    x = np.asarray([r["features"] for r in records], dtype=float)
    y = np.asarray([r["late"] for r in records], dtype=int)
    development, test_ids = train_test_split(np.arange(300), test_size=.2, stratify=y, random_state=17)
    train_ids, validation_ids = train_test_split(development, test_size=.25,
        stratify=y[development], random_state=19)
    assert not set(train_ids) & set(validation_ids) and not set(development) & set(test_ids)
    model = Pipeline([("scale", StandardScaler()),
        ("classifier", LogisticRegression(C=1, max_iter=1000, random_state=0))])
    model.fit(x[train_ids], y[train_ids])
    validation_probability = model.predict_proba(x[validation_ids])[:, 1]
    candidates = [{"threshold": t, **metrics(y[validation_ids].tolist(),
        (validation_probability >= t).astype(int).tolist())} for t in thresholds]
    choice = min(candidates, key=lambda r: (r["cost_FN5_FP1"], -r["threshold"]))
    constant_candidates = [{"constant": c, **metrics(y[validation_ids].tolist(),
        [c] * len(validation_ids))} for c in (0, 1)]
    baseline_choice = min(constant_candidates, key=lambda r: (r["cost_FN5_FP1"], r["constant"]))
    # Freeze the model, threshold and baseline choices. No refit after validation or test.
    test_probability = model.predict_proba(x[test_ids])[:, 1]
    test_prediction = (test_probability >= choice["threshold"]).astype(int).tolist()
    test_result = metrics(y[test_ids].tolist(), test_prediction)
    baseline_test = metrics(y[test_ids].tolist(), [baseline_choice["constant"]] * len(test_ids))
    accepted = (test_result["cost_FN5_FP1"] < baseline_test["cost_FN5_FP1"]
                and test_result["recall"] is not None and test_result["recall"] >= minimum_recall)
    result = {"scope": "Original synthetic order-review exercise; no real orders, intervention or API deployment",
        "predeclared": {"cost_false_negative": 5, "cost_false_positive": 1,
            "threshold_candidates": thresholds, "threshold_tie_rule": "higher threshold if same validation cost",
            "minimum_test_recall": minimum_recall,
            "acceptance": "test cost strictly below validation-selected constant baseline AND recall>=.8",
            "feature_order_at_creation": ["distance", "queue", "promised_days", "fragile"]},
        "versions": {"python": platform.python_version(), "numpy": np.__version__, "sklearn": sklearn.__version__},
        "data_sha256": hashlib.sha256(json.dumps(records, sort_keys=True, separators=(",", ":")).encode()).hexdigest(),
        "split_ids": {"train": train_ids.tolist(), "validation": validation_ids.tolist(), "test": test_ids.tolist()},
        "training": {"count": len(train_ids), "scaler_n_samples_seen": int(model.named_steps["scale"].n_samples_seen_),
            "scaler_mean": model.named_steps["scale"].mean_.tolist(),
            "scaler_scale": model.named_steps["scale"].scale_.tolist(),
            "coefficients": model.named_steps["classifier"].coef_[0].tolist(),
            "intercept": float(model.named_steps["classifier"].intercept_[0])},
        "validation_candidates": candidates, "validation_choice": choice,
        "validation_constant_baselines": constant_candidates, "baseline_choice": baseline_choice,
        "frozen_test": test_result, "baseline_test": baseline_test,
        "test_predictions": [{"id": int(i), "label": int(y[i]), "probability": float(p), "decision": int(a)}
            for i, p, a in zip(test_ids, test_probability, test_prediction)],
        "acceptance_passed": bool(accepted),
        "limits": ["Cost units and .8 recall are teaching assumptions, not observed business economics.",
            "A model score does not prove that human review prevents a late delivery or produces savings.",
            "Random synthetic split assumes this constructed population; real deployment may require temporal/group splits.",
            "One held-out test was used after fixed validation selection; no test-based retuning or refit.",
            "No calibration, real-world fairness, privacy compliance, capacity constraint or confidence interval was established.",
            "Passing this toy acceptance rule is not authorization for production rollout."]}
    print(json.dumps(result, indent=2, allow_nan=False))


if __name__ == "__main__":
    main()

DS72 image sources and modifications

Figure 1 and cover: Alexander Schröder, CRISP DM Data mining management process (2021). Original work page: https://commons.wikimedia.org/wiki/File:CRISP_DM_Data_mining_management_process.jpg ; source revision: https://commons.wikimedia.org/w/index.php?title=File:CRISP_DM_Data_mining_management_process.jpg&oldid=844742088 . Licensed CC BY-SA 4.0, https://creativecommons.org/licenses/by-sa/4.0/ . Changes: source JPEG converted to RGB PNG without content edits; the complete image is proportionally fitted to 876x900 on a white 1600x900 canvas for the cover. These image adaptations are distributed under CC BY-SA 4.0. No labels, values or arrows were added. This is the uploader's typical illustration of six CRISP-DM phases and feedback relationships, not the normative specification or this project's actual deployment.

Figure 2: The scikit-learn developers, Visualizing cross-validation behavior in scikit-learn, official 1.9.1 documentation, https://scikit-learn.org/1.9/auto_examples/model_selection/plot_cv_indices.html . The official KFold PNG is reproduced byte-for-byte. It depicts four splits over 100 synthetic sample indices and separate class/group rows. KFold in this source does not use those class/group labels to stratify or isolate groups; the generated groups have unequal sizes. This is not this project's 180/60/60 train/validation/test holdout. BSD 3-Clause; full original COPYING follows.

Figure 3: The scikit-learn developers, Release Highlights for scikit-learn 1.9, Computing metrics across thresholds, official 1.9.1 documentation, https://scikit-learn.org/1.9/auto_examples/release_highlights/plot_release_highlights_1_9_0.html . The original synthetic-data Accuracy/F1 plot is reproduced byte-for-byte. The source uses make_classification(weights=[.9,.1],random_state=0), fits LogisticRegression and evaluates threshold metrics on the same X/y. It is an in-sample API illustration, not untouched-test performance or this project's 5FN+FP cost/recall acceptance. The source figure illustrates the threshold concept using a current API; this project itself ran scikit-learn 1.5.1 and did not execute the 1.9 example or claim that API existed before 2023. BSD 3-Clause; full original COPYING follows.

Original scikit-learn 1.9.1 BSD 3-Clause license (verbatim)
BSD 3-Clause License

Copyright (c) 2007-2026 The scikit-learn developers.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.