Build a Generative AI Model

Table of Contents

1. Module 1 — Building and Training Autoencoders
2. Module 2 — From Autoencoders to GANs: Building Adversarial Generative Models
3. Appendix — Installation Commands Summary

Business Use Case: Globomantics

Throughout this course, we work with package telemetry data for the fictional company Globomantics. The core problem: the company has lots of unlabeled transport data, but very few labeled examples of damaged packages. The goal is to learn useful representations without labels, then generate realistic synthetic data.

Shipment Features

Module 1 — Building and Training Autoencoders

1.1 What Autoencoders Are and Why They Matter in Generative AI

Core Concept

An autoencoder is a model that learns useful representations without labels by reconstructing its own inputs. The key idea is the bottleneck (or latent space): a compressed representation capturing the essential structure of the data.

flowchart LR
    subgraph INPUT["Input (9 features)"]
        X["x\nweight, length, width,\nheight, transit_time,\ntemp, humidity,\nshock_events, value"]
    end
    subgraph ENCODER["Encoder"]
        E1["Linear(9→16)\n+ ReLU"]
        E2["Linear(16→latent_dim)"]
        E1 --> E2
    end
    subgraph BOTTLENECK["Bottleneck / Latent Space"]
        Z["z\n(2D or 3D)"]
    end
    subgraph DECODER["Decoder"]
        D1["Linear(latent_dim→16)\n+ ReLU"]
        D2["Linear(16→9)"]
        D1 --> D2
    end
    subgraph OUTPUT["Reconstructed Output"]
        XHAT["x̂\n(reconstruction)"]
    end

    X --> E1
    E2 --> Z
    Z --> D1
    D2 --> XHAT

Why Autoencoder for Globomantics?

• Abundant unlabeled telemetry data
• Few anomaly examples → supervised learning is difficult
• Autoencoder learns to reconstruct normal shipments → anomalies will have high reconstruction error

Basic Architecture

class ShipmentAutoencoder(nn.Module):
    def __init__(self, input_dim: int = 9, latent_dim: int = 2):
        super().__init__()
        self.encoder = nn.Sequential(
            nn.Linear(input_dim, 16),
            nn.ReLU(),
            nn.Linear(16, latent_dim),
        )
        self.decoder = nn.Sequential(
            nn.Linear(latent_dim, 16),
            nn.ReLU(),
            nn.Linear(16, input_dim),
        )

    def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
        z = self.encoder(x)
        x_hat = self.decoder(z)
        return x_hat, z  # reconstruction AND latent vector

Synthetic Data Generation

def generate_shipments(n_samples: int = 800) -> torch.Tensor:
    weight    = np.random.normal(10, 3, n_samples)
    length    = np.random.normal(40, 10, n_samples)
    width     = np.random.normal(30, 8, n_samples)
    height    = np.random.normal(20, 6, n_samples)
    transit   = np.random.normal(72, 24, n_samples)
    temp      = np.random.normal(15, 10, n_samples)
    humidity  = np.random.normal(50, 20, n_samples)
    shocks    = np.random.poisson(2, n_samples)
    value     = np.random.normal(500, 200, n_samples)

    X = np.stack([weight, length, width, height, transit,
                  temp, humidity, shocks, value], axis=1).astype(np.float32)
    return torch.tensor(X, dtype=torch.float32)

1.2 Implementing an Autoencoder from Scratch — Part 1

Feature Standardization

def standardize_features(X: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    mu    = X.mean(dim=0, keepdim=True)
    sigma = X.std(dim=0, keepdim=True).clamp_min(1e-6)
    Xz    = (X - mu) / sigma
    return Xz, mu, sigma

Reconstruction Loss

def reconstruction_loss(x_hat: torch.Tensor, x: torch.Tensor, loss_type: str = "mse") -> torch.Tensor:
    if loss_type == "mse":
        return F.mse_loss(x_hat, x)  # penalizes large errors heavily
    if loss_type == "mae":
        return F.l1_loss(x_hat, x)   # less sensitive to outliers
    raise ValueError("loss_type must be 'mse' or 'mae'")

Main Training Loop

def train_autoencoder(model, X_train, cfg):
    model.train()
    opt = torch.optim.Adam(model.parameters(), lr=cfg.lr)

    for epoch in range(cfg.epochs):
        perm = torch.randperm(X_train.size(0))
        X_shuffled = X_train[perm]

        for start in range(0, X_train.size(0), cfg.batch_size):
            batch = X_shuffled[start:start + cfg.batch_size]

            x_hat, z = model(batch)
            loss = reconstruction_loss(x_hat, batch, cfg.loss_type)

            opt.zero_grad(set_to_none=True)
            loss.backward()
            opt.step()

1.4 Training Stability Techniques

mindmap
  root((Training Stability))
    Weight Decay
      Penalizes large weights
      L2 regularization
    Gradient Clipping
      Limits update magnitude
      Prevents gradient explosion
      clip_grad_norm = 1.0
    Early Stopping
      Stops when improvement < threshold
      early_stop_delta = 1e-4

@dataclass
class TrainConfig:
    epochs:           int   = 2
    batch_size:       int   = 256
    lr:               float = 1e-3
    loss_type:        str   = "mse"
    weight_decay:     float = 0.0
    clip_grad_norm:   float = 1.0
    early_stop_delta: float = 1e-4

1.5 Visualizing Latent Space and Reconstructed Outputs

PCA to Reduce Latent Space to 2D

def pca_to_2d(Z: torch.Tensor) -> np.ndarray:
    z = Z.detach().cpu().numpy()
    z = z - z.mean(axis=0, keepdims=True)
    cov = np.cov(z, rowvar=False)
    eigvals, eigvecs = np.linalg.eigh(cov)
    order = np.argsort(eigvals)[::-1]
    top2  = eigvecs[:, order[:2]]
    return z @ top2

Reconstruction Error (Anomaly Detection)

@torch.no_grad()
def reconstruction_errors(model, X):
    x_hat, _ = model(X)
    per_row = ((x_hat - X) ** 2).mean(dim=1)
    return per_row

errs  = reconstruction_errors(model, X)
worst = torch.topk(errs, k=5).indices   # 5 potential anomalies
best  = torch.topk(-errs, k=5).indices  # 5 "typical" shipments

Latent Walk — Latent Space Perturbation

@torch.no_grad()
def latent_perturbation_demo(model, X, idx=0, steps=5, scale=0.75):
    x0   = X[idx:idx + 1]
    z0   = model.encoder(x0)
    dim  = 0

    deltas = torch.linspace(-scale, scale, steps).view(-1, 1)
    Z      = z0.repeat(steps, 1)
    Z[:, dim:dim + 1] += deltas

    X_var = model.decoder(Z)
    return X_var

1.6 Autoencoder Performance Evaluation

@torch.no_grad()
def per_feature_r2(model, X):
    x_hat, _ = model(X)
    sse = ((X - x_hat) ** 2).sum(dim=0)
    sst = ((X - X.mean(dim=0)) ** 2).sum(dim=0).clamp_min(1e-9)
    return 1.0 - (sse / sst)  # R² per feature (1.0 = perfect)

Module 2 — From Autoencoders to GANs

2.1 Why GANs? Introduction to Adversarial Generative Modeling

A basic autoencoder can reconstruct existing data, but sampling from the latent space doesn’t guarantee realistic results since the latent space distribution isn’t constrained.

The Adversarial Idea of GANs

flowchart LR
    NOISE["Random noise\nz ~ N(0,1)"]
    G["Generator G\n(creates fakes)"]
    FAKE["Fake shipment"]
    REAL["Real shipment"]
    D["Discriminator D\n(detects fakes)"]
    PROB["Probability\n0 = fake\n1 = real"]

    NOISE --> G --> FAKE --> D --> PROB
    REAL --> D

GAN training = minimax: $$\min_G \max_D ; \mathbb{E}[\log D(x_{real})] + \mathbb{E}[\log(1 - D(G(z)))]$$

2.2 Implementing the Generator and Discriminator

class Generator(nn.Module):
    def __init__(self, z_dim: int = 5, output_dim: int = 9, hidden: int = 32):
        super().__init__()
        self.z_dim = z_dim
        self.net = nn.Sequential(
            nn.Linear(z_dim, hidden),
            nn.ReLU(),
            nn.Linear(hidden, hidden),
            nn.ReLU(),
            nn.Linear(hidden, output_dim),  # linear output (standardized features)
        )

    def forward(self, z: torch.Tensor) -> torch.Tensor:
        return self.net(z)

class Discriminator(nn.Module):
    def __init__(self, input_dim: int = 9, hidden: int = 32):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(input_dim, hidden),
            nn.ReLU(),
            nn.Linear(hidden, hidden // 2),
            nn.ReLU(),
            nn.Linear(hidden // 2, 1),
            nn.Sigmoid(),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.net(x)

2.3 Training a GAN: Loss Functions, Instability, and Practical Fixes

GAN Stability Techniques

mindmap
  root((GAN Stability))
    Label Smoothing
      real_label = 0.9 instead of 1.0
      Avoids D overconfidence
    Label Flipping
      flip_prob = 0.02
      Slightly perturbs D
    Instance Noise
      Adds Gaussian noise to D inputs
      std = 0.05
    Gradient Clipping
      clip_grad_norm = 1.0
    LR Scheduling
      StepLR: reduces lr midway
      gamma = 0.7
    Adam Optimizer
      betas = (0.5, 0.999)

Complete GAN Training Loop

def train_gan(G, D, X, cfg):
    opt_d = torch.optim.Adam(D.parameters(), lr=cfg.lr_d, betas=(0.5, 0.999))
    opt_g = torch.optim.Adam(G.parameters(), lr=cfg.lr_g, betas=(0.5, 0.999))
    bce   = nn.BCEWithLogitsLoss()

    for step in range(1, cfg.steps + 1):
        # Step 1: Update Discriminator
        real = sample_real_batch(X, cfg.batch_size)
        z    = torch.randn(cfg.batch_size, cfg.z_dim)
        fake = G(z).detach()  # .detach(): don't touch G here

        y_real = torch.full((cfg.batch_size, 1), cfg.real_label)
        y_fake = torch.zeros(cfg.batch_size, 1)

        d_loss = bce(D(real), y_real) + bce(D(fake), y_fake)
        opt_d.zero_grad(set_to_none=True)
        d_loss.backward()
        nn.utils.clip_grad_norm_(D.parameters(), cfg.clip_grad_norm)
        opt_d.step()

        # Step 2: Update Generator
        z    = torch.randn(cfg.batch_size, cfg.z_dim)
        fake = G(z)
        y_gen = torch.full((cfg.batch_size, 1), cfg.real_label)

        g_loss = bce(D(fake), y_gen)
        opt_g.zero_grad(set_to_none=True)
        g_loss.backward()
        nn.utils.clip_grad_norm_(G.parameters(), cfg.clip_grad_norm)
        opt_g.step()

2.4 Generating Samples and Understanding GAN Output Quality

Quality Metrics

2.5 Ethics, Risks, and Practical Considerations

Appendix — Installation Commands

pip install torch numpy matplotlib

Demo Files Summary

Key Glossary

Search Terms

generative · ai · model · deep · neural · networks · machine · data · science · gan · autoencoder · autoencoders · gans · latent · space · adversarial · globomantics · loop · loss · practical · quality · reconstruction · stability · techniques