LSTM 時序預測：什麼時候值得用深度模型？

c(t) = f(t) * c(t-1) + i(t) * g(t)
h(t) = o(t) * tanh(c(t))

f、i、o：sigmoid gates，逐元素控制
 g：tanh candidate，提出要寫入的內容
 c：cell state
 h：hidden state

LSTM cell：
  輸入權重 4*8*1 = 32
  hidden 權重 4*8*8 = 256
  兩組 bias 2*(4*8) = 64
  cell 合計 352
線性 head：8*1 + 1 = 9
總計：361

MLP：14 → 16 → 1，中間 Tanh
  14*16 + 16 + 16*1 + 1 = 257

單一樣本：過去14點 → 下一點
訓練 x：batch × 14 × 1
訓練 y：batch

LSTM output：batch × 14 × 8
LSTM h_n/c_n：1 × batch × 8
線性 head：取最後時間步的8維hidden → 1個預測

z(t) = [y(t) - train_mean] / train_population_std
反轉：ŷ(t) = ẑ(t) * train_population_std + train_mean

Python 3.10.12
torch 2.3.1+cu118
CPU / float64 / single thread

python3 lstm_and_mlp_forecasts.py > stdout.json

torch==2.3.1

"""Actual CPU LSTM/MLP fits on an original deterministic teaching sequence.

All models forecast the same 14 future points from the same observed prefix.
Neural networks see 14 lag values; both reset state for every input window.
"""
import json
import platform

import torch
from torch import nn


WINDOW = 14
HORIZON = 14
EPOCHS = 200
SEEDS = (7, 19, 41)
MODELS = ("seasonal_naive", "mlp", "lstm")
PATTERN = (-6, -3, 0, 2, 5, 11, 8)
DISTURBANCE = (-1, 0, 1, 0, -1)
SERIES = [50 + 0.1 * t + PATTERN[t % 7] + DISTURBANCE[t % 5]
          for t in range(168)]


class ForecastNet(nn.Module):
    def __init__(self, kind):
        super().__init__()
        self.kind = kind
        if kind == "lstm":
            self.recurrent = nn.LSTM(1, 8, batch_first=True)
            self.head = nn.Linear(8, 1)
        else:
            assert kind == "mlp"
            self.mlp = nn.Sequential(nn.Flatten(), nn.Linear(WINDOW, 16),
                                     nn.Tanh(), nn.Linear(16, 1))

    def forward(self, inputs):
        if self.kind == "mlp":
            return self.mlp(inputs).squeeze(-1)
        # No h0/c0 supplied: each window starts from zeros, no state carryover.
        outputs, _ = self.recurrent(inputs)
        return self.head(outputs[:, -1, :]).squeeze(-1)


def neural_forecast(history, kind, seed):
    torch.manual_seed(seed)
    raw = torch.tensor(history, dtype=torch.float64)
    center = raw.mean()
    scale = raw.std(correction=0)
    assert scale > 0
    normalized = (raw - center) / scale
    x = torch.stack([normalized[t-WINDOW:t] for t in range(WINDOW, len(raw))])
    x = x.unsqueeze(-1)
    y = normalized[WINDOW:]
    assert x.shape == (len(history) - WINDOW, WINDOW, 1)
    net = ForecastNet(kind).double()
    optimizer = torch.optim.Adam(net.parameters(), lr=0.01)
    net.train()
    initial_loss = float(nn.functional.mse_loss(net(x), y).detach())
    for _ in range(EPOCHS):
        optimizer.zero_grad()
        loss = nn.functional.mse_loss(net(x), y)
        loss.backward()
        optimizer.step()
    net.eval()
    with torch.no_grad():
        final_loss = float(nn.functional.mse_loss(net(x), y))
        prefix = normalized.tolist()
        forecast = []
        for _ in range(HORIZON):
            window = torch.tensor(prefix[-WINDOW:], dtype=torch.float64).reshape(1, WINDOW, 1)
            value = float(net(window)[0])
            prefix.append(value)
            forecast.append(value * float(scale) + float(center))
    assert torch.isfinite(torch.tensor(forecast)).all()
    return forecast, {"seed": seed, "training_target_indices": [WINDOW, len(history)-1],
        "input_shape": list(x.shape), "train_only_mean": float(center),
        "train_only_population_std": float(scale),
        "parameter_count": sum(p.numel() for p in net.parameters()),
        "initial_scaled_training_MSE": initial_loss, "final_scaled_training_MSE": final_loss}


def mae(truth, prediction):
    return sum(abs(a-b) for a, b in zip(truth, prediction)) / len(truth)


def evaluate(origin):
    history = SERIES[:origin]
    truth = SERIES[origin:origin + HORIZON]
    rows = []
    for kind in MODELS:
        seeds = (None,) if kind == "seasonal_naive" else SEEDS
        for seed in seeds:
            if seed is None:
                predicted = [history[-7 + h % 7] for h in range(HORIZON)]
                metadata = {"seed": None, "parameter_count": 0}
            else:
                predicted, metadata = neural_forecast(history, kind, seed)
            rows.append({"origin": origin, "forecast_indices": [origin, origin+HORIZON-1],
                "model": kind, **metadata, "prediction": [round(v, 8) for v in predicted],
                "truth": [round(v, 8) for v in truth], "MAE": mae(truth, predicted)})
    return rows


def main():
    torch.set_num_threads(1)
    torch.use_deterministic_algorithms(True)
    validation = evaluate(126) + evaluate(140)
    scores = {kind: sum(r["MAE"] for r in validation if r["model"] == kind)
              / sum(r["model"] == kind for r in validation) for kind in MODELS}
    selected = min(MODELS, key=lambda kind: (scores[kind], MODELS.index(kind)))
    # Freeze model-family selection before examining final-block errors.
    final = evaluate(154)
    final_scores = {kind: sum(r["MAE"] for r in final if r["model"] == kind)
                    / sum(r["model"] == kind for r in final) for kind in MODELS}
    print(json.dumps({"scope": "Hand-set 168-point daily signal, actual CPU float64 training. "
        "Fixed architectures/200 updates/three initialization seeds, no external benchmark.",
        "versions": {"python": platform.python_version(), "torch": torch.__version__},
        "device": "cpu", "dtype": "float64", "threads": torch.get_num_threads(),
        "window": WINDOW, "horizon": HORIZON, "epochs": EPOCHS, "seeds": SEEDS,
        "series": [round(v, 8) for v in SERIES], "validation": validation,
        "mean_validation_MAE_across_origins_and_seeds": scores,
        "selected_before_final_evaluation": selected, "final": final,
        "final_mean_individual_seed_MAE": final_scores,
        "limits": ["The mean of individual seed MAEs is not an ensemble forecast MAE.",
            "Three initialization seeds are not independent datasets or a significance test.",
            "Training histories overlap; 14-day blocks never receive intermediate actuals.",
            "Scalers use training-prefix observations only; no final-block scaling statistics.",
            "Only 200 fixed updates, not proof of optimizer convergence or fair tuning budgets.",
            "LSTM and MLP have different parameter counts but identical observed lag inputs.",
            "State resets for each window; no claim of learning beyond the 14-value context.",
            "Same-environment deterministic settings do not promise cross-platform byte identity.",
            "No empirical real-data performance, calibrated intervals, or general architecture ranking."]},
        ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()

{
  "mean_validation_MAE_across_origins_and_seeds": {
    "seasonal_naive": 1.514285714285713,
    "mlp": 0.4754750894753413,
    "lstm": 1.3062758667841428
  },
  "selected_before_final_evaluation": "mlp"
}

Source image credits and license

Four sequence source figures and one selected LSTM gate figure are from Dive into Deep Learning, English edition, by Aston Zhang, Zachary C. Lipton, Mu Li, and Alexander J. Smola. The book visuals are licensed under Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0): https://creativecommons.org/licenses/by-sa/4.0/ . The sample-code license is separate and is not relied upon for these figures.

The four sequence SVGs were obtained from the live D2L 1.0.3 chapter image URLs and locally hash-pinned. Repository revision 23d7a5aecceee57d1292c56e90cce307f183bb0a identifies the chapter Markdown, README and license; these rendered output SVGs are not claimed to be files in that commit. The selected LSTM gate image is rendered from the pinned D2L LSTM source SVG.

Changes: the original source SVGs are rendered on white at 1400px width; two forecast mobile derivatives show the complete chart first, followed by enlarged original plot-interior and complete-legend crops; the cover fits the complete one-step-vs-recursive plot proportionally to a white 1600x900 canvas. No data, labels, lines, arrows or chart elements were added or redrawn. Shared adaptations are offered under CC BY-SA 4.0. No additional restrictions are applied to recipients exercising the license rights.

The D2L sequence forecast example is synthetic noisy-sine data fitted with linear regression, not an LSTM result.