AutoML & Neural Architecture Search

Reviewed & published by Brayan K

Automate model selection, hyperparameter tuning, and even neural network architecture design.

Part of the free AI & Machine Learning course at LearnCodingFast — hands-on lessons with examples you run in your browser, plus practice exercises and a quick quiz.

What You'll Learn in This Lesson

• • How AutoML searches across algorithms and hyperparameters
• • Bayesian optimization for smarter hyperparameter tuning
• • Neural Architecture Search (NAS) techniques: RL, evolutionary, DARTS
• • When to use AutoML vs manual tuning

import numpy as np

# ============================================
# AUTOMATED MODEL SELECTION (AutoML)
# ============================================
np.random.seed(42)

print("=== AutoML: Let the Machine Choose the Model ===")
print()
print("Instead of manually trying models, AutoML systematically")
print("searches the space of algorithms and hyperparameters.")
print()

class AutoMLSearch:
    """Simplified AutoML pipeline that searches models + hyperparameters."""
    def __init__(self):
        self.results = []
    
    def search(self, X_size, n_features):
        models = [
            {"name": "LogisticRegression", "params": {"C": [0.01, 0.1, 1, 10]}},
            {"name": "RandomForest", "params": {"n_estimators": [50, 100, 200], "max_depth": [5, 10, None]}},
            {"name": "XGBoost", "params": {"lr": [0.01, 0.05, 0.1], "n_estimators": [100, 200, 500]}},
            {"name": "SVM", "params": {"C": [0.1, 1, 10], "kernel": ["rbf", "linear"]}},
            {"name": "NeuralNet", "params": {"hidden": [64, 128, 256], "layers": [2, 3]}},
        ]
        
        print(f"Dataset: {X_size} samples, {n_features} features")
        print(f"Search space: {len(models)} algorithms")
        print()
        
        total_configs = 0
        for model in models:
            n_configs = 1
            for values in model["params"].values():
                n_configs *= len(values)
            total_configs += n_configs
        
        print(f"Total configurations to try: {total_configs}")
        print()
        print("Running search...")
        print()
        
        for model in models:
            # Simulate trying different hyperparameters
            best_score = 0
            best_params = {}
            
            # Generate param combinations (simplified)
            keys = list(model["params"].keys())
            for val in model["params"][keys[0]]:
                score = np.random.uniform(0.75, 0.95)
                if score > best_score:
                    best_score = score
                    best_params = {keys[0]: val}
            
            self.results.append({
                "model": model["name"],
                "score": best_score,
                "params": best_params,
            })
        
        self.results.sort(key=lambda x: x["score"], reverse=True)
        
        print(f"{'Rank':<6s} {'Model':<20s} {'CV Score':>9s} {'Best Params'}")
        print("-" * 65)
        for i, r in enumerate(self.results):
            medal = "🥇" if i == 0 else "🥈" if i == 1 else "🥉" if i == 2 else "  "
            print(f" {medal} {i+1}  {r['model']:<20s} {r['score']:>8.4f}  {r['params']}")
        
        print()
        winner = self.results[0]
        print(f"🏆 Winner: {winner['model']} (score: {winner['score']:.4f})")
        return winner

automl = AutoMLSearch()
best = automl.search(X_size=10000, n_features=25)

print()
print("=== Bayesian Optimization ===")
print()
print("Random search wastes time on bad regions.")
print("Bayesian optimization is smarter — it builds a model")
print("of the objective function and focuses on promising areas.")
print()

# Simulate Bayesian optimization
def objective(x):
    """Black-box function to optimize (simulated model accuracy)."""
    return -(x - 0.3)**2 + 0.95 + np.random.normal(0, 0.01)

print("Bayesian Optimization Trace:")
print(f"  {'Iter':<6s} {'x':>6s} {'f(x)':>8s} {'Best':>8s}")
print("  " + "-" * 32)

best_x, best_y = None, -999
for i in range(10):
    if i < 3:
        x = np.random.uniform(0, 1)  # Explore first
    else:
        x = best_x + np.random.normal(0, 0.1)  # Exploit near best
        x = np.clip(x, 0, 1)
    
    y = objective(x)
    if y > best_y:
        best_x, best_y = x, y
    
    marker = " ⭐ new best!" if y == best_y else ""
    print(f"  {i+1:<6d} {x:>6.3f} {y:>8.4f} {best_y:>8.4f}{marker}")

print(f"\n  Optimal x ≈ {best_x:.3f} (true optimum is 0.300)")

import numpy as np

# ============================================
# NEURAL ARCHITECTURE SEARCH (NAS)
# ============================================
np.random.seed(42)

print("=== Neural Architecture Search ===")
print()
print("NAS automates the design of neural network architectures.")
print("Instead of humans designing layers, the algorithm discovers them.")
print()
print("Famous NAS results:")
print("  • EfficientNet: Found by NAS, beats hand-designed ResNets")
print("  • NASNet: Google discovered novel cell architectures")
print("  • DARTS: Differentiable NAS (much faster than RL-based)")
print()

# Simulate architecture search space
operations = ["conv_3x3", "conv_5x5", "max_pool", "avg_pool", "skip_connect", "sep_conv_3x3"]
n_layers = 5
n_cells = 3

print(f"Search Space:")
print(f"  Operations per edge: {len(operations)}")
print(f"  Layers: {n_layers}")
print(f"  Cells per layer: {n_cells}")
print(f"  Total architectures: ~{len(operations)**(n_layers * n_cells):,}")
print()

# Simulate NAS search
class NASSearch:
    def __init__(self):
        self.population = []
    
    def random_architecture(self):
        arch = []
        for l in range(n_layers):
            cell = []
            for c in range(n_cells):
                op = np.random.choice(operations)
                cell.append(op)
            arch.append(cell)
        return arch
    
    def evaluate(self, arch):
        """Simulate training and evaluating architecture."""
        score = 0.80
        for layer in arch:
            for op in layer:
                if "conv" in op: score += np.random.uniform(0.005, 0.015)
                elif "sep_conv" in op: score += np.random.uniform(0.008, 0.018)
                elif "skip" in op: score += np.random.uniform(0.003, 0.01)
                else: score += np.random.uniform(0, 0.005)
        score += np.random.normal(0, 0.01)
        return min(score, 0.99)
    
    def arch_to_string(self, arch):
        return " → ".join(["+".join([op[:4] for op in cell]) for cell in arch[:3]]) + "..."
    
    def search(self, n_trials=8):
        print("Searching architectures...")
        print()
        
        for i in range(n_trials):
            arch = self.random_architecture()
            score = self.evaluate(arch)
            self.population.append({"arch": arch, "score": score})
            print(f"  Trial {i+1:2d}: {self.arch_to_string(arch)}")
            print(f"           Score: {score:.4f}")
        
        self.population.sort(key=lambda x: x["score"], reverse=True)
        print()
        print("=== Top 3 Architectures ===")
        for i in range(3):
            p = self.population[i]
            medal = ["🥇", "🥈", "🥉"][i]
            print(f"  {medal} Score: {p['score']:.4f}")
            print(f"     Layers:")
            for l, cell in enumerate(p["arch"]):
                print(f"       L{l}: {' + '.join(cell)}")

nas = NASSearch()
nas.search()

print()
print("=== NAS Approaches Compared ===")
print()
print(f"  {'Method':<20s} {'Speed':>8s} {'Quality':>8s} {'GPU Hours':>10s}")
print("  " + "-" * 48)
methods = [
    ("RL-based (NASNet)", "Slow", "Best", "~22,400"),
    ("Evolutionary", "Medium", "Good", "~3,000"),
    ("DARTS", "Fast", "Good", "~1.5"),
    ("One-Shot (ENAS)", "Very Fast", "Good", "~0.5"),
]
for name, speed, quality, gpu in methods:
    print(f"  {name:<20s} {speed:>8s} {quality:>8s} {gpu:>10s}")

print()
print("💡 Start with DARTS or ENAS for practical NAS.")
print("   Full RL-based NAS requires massive compute budgets.")

🔍 Worked example: the loop that IS AutoML

The sections above describe what AutoML does. This is the thing itself, small enough to read in one go: a grid of settings, a model fitted under each, one honest score on data the model never saw, and the best kept. Run it and look at the table before the answer — the worst configuration is not the one you would guess.

# WORKED EXAMPLE — AutoML, stripped to its skeleton: try every combination of
# settings, score each one the same way, keep the best. No library, no magic.

# A line to learn: y = 2x + 1, with a little deterministic noise.
train_x = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
train_y = [1.20, 1.38, 1.62, 1.80, 2.02, 2.18]
val_x   = [0.7, 0.8, 0.9]
val_y   = [2.42, 2.58, 2.81]

def fit(learning_rate, epochs):
    w, b = 0.0, 0.0                                  # always start from the same place
    for _ in range(epochs):
        dw = db = 0.0
        for x, y in zip(train_x, train_y):
            err = (w * x + b) - y
            dw += 2 * err * x / len(train_x)
            db += 2 * err / len(train_x)
        w -= learning_rate * dw
        b -= learning_rate * db
    return w, b

def mse(w, b):
    return sum(((w * x + b) - y) ** 2 for x, y in zip(val_x, val_y)) / len(val_x)

# THE SEARCH SPACE: 3 learning rates x 2 epoch counts = 6 configurations
grid = [(lr, ep) for lr in (0.001, 0.01, 0.05) for ep in (10, 100)]

best = None
print(f"{'lr':>6} {'epochs':>6} {'val MSE':>9}")
for lr, ep in grid:
    w, b = fit(lr, ep)
    score = mse(w, b)
    print(f"{lr:>6} {ep:>6} {score:>9.4f}")
    if best is None or score < best[0]:
        best = (score, lr, ep)

print()
print(f"best: lr={best[1]}, epochs={best[2]}  (val MSE {best[0]:.4f})")
print("That loop IS AutoML. Real tools search bigger spaces more cleverly,")
print("but the shape - candidates, one scoring rule, keep the winner - is this.")

# ✅ Expected output:
#     lr epochs   val MSE
#  0.001     10    6.5754
#  0.001    100    4.8827
#   0.01     10    4.8667
#   0.01    100    0.6730
#   0.05     10    1.5065
#   0.05    100    0.2553
#
# best: lr=0.05, epochs=100  (val MSE 0.2553)
# That loop IS AutoML. Real tools search bigger spaces more cleverly,
# but the shape - candidates, one scoring rule, keep the winner - is this.

🎯 Your turn: put the module together

Now run a search of your own, on a forecasting problem instead of a line. The structure is identical to the worked example — candidates, a scoring rule, keep the winner — and the printed scores let you check that the winner really did win.

# 🎯 YOUR TURN — automate a smaller search: which moving-average window
# forecasts this series best? Score each window by mean absolute error and
# keep the lowest. Fill in the three ___ blanks.

series = [10, 12, 11, 13, 12, 14, 13, 15, 14, 16]

def forecast_errors(window):
    errors = []
    for i in range(window, len(series)):
        pred = sum(series[i - window:i]) / window     # average of the last `window` values
        # 1) the error is the gap between the prediction and the real value
        errors.append(abs(pred - ___))
        # 👉 replace ___ with series[i]
    return sum(errors) / len(errors)

best_window, best_mae = None, None
# 2) the candidates to try
for window in [___]:
    # 👉 replace ___ with 1, 2, 3
    score = forecast_errors(window)
    print(f"window={window}  MAE={score:.3f}")
    # 3) lower error is better, so a candidate wins when its score is smaller
    if best_mae is None or score ___ best_mae:
        # 👉 replace ___ with <
        best_window, best_mae = window, score

print(f"best window: {best_window}")

# ✅ Expected output:
# window=1  MAE=1.556
# window=2  MAE=0.750
# window=3  MAE=1.143
# best window: 2

⚠️ Common Mistakes

📋 Quick Reference — AutoML Tools

🎉 Lesson Complete!

You've mastered AutoML and NAS! Next, learn the critical topic of Ethical AI, bias mitigation, and responsible ML.

Practice quiz

Continue this course

• Previous: Graph Neural Networks (GNNs) for Social & Knowledge Graphs
• Next: Ethical AI, Bias Mitigation & Safety Principles in ML — Identify, measure, and reduce bias — build fair and responsible AI systems
• Quick reference: AI & Machine Learning cheat sheet