Skip to content

Latest commit

 

History

History
238 lines (195 loc) · 4.88 KB

README.md

File metadata and controls

238 lines (195 loc) · 4.88 KB
Raw

Tiny AutoDiff

A tiny autograd library. Implements backpropagation autodiff. It supports all you need to build small neural networks.

Library

Add library to your project using DUB:

dub add tiny-autodiff

Precision

Use the versions configuration to specify the precision:

  • TAUTODIFF_USE_FLOAT
  • TAUTODIFF_USE_DOUBLE
  • TAUTODIFF_USE_REAL
// dub.sdl
versions "TAUTODIFF_USE_FLOAT"

// dub.json
versions: ["TAUTODIFF_USE_FLOAT"]

Example usage

Value

import rk.tautodiff;

auto a = value(2);
auto b = value(-3);
auto c = value(10);
auto f = value(-2);
auto e = a * b;
auto d = e + c;
auto g = f * d;

// backward
g.backward();

// check grad after backward
assert(g.grad == 1);
assert(f.grad == 4);
assert(d.grad == -2);
assert(e.grad == -2);
assert(c.grad == -2);
assert(b.grad == -4);
assert(a.grad == 6);

ChainSolver

Use ChainSolver to solve equations step by step.

import rk.tautodiff;

// create solver
auto solver = ChainSolver(0); // 0 is initial value

// operations using the produced result 
solver += 5; // 0 + 5 = 5
solver *= 2; // 3 * 2 = 6

// append new value and work with it
solver ~= solver / value(2);
assert(solver.data == 3);

// backward
solver.backward();
assert(solver.grad == 1);

// zero grad
solver.zeroGrad();
assert(solver.grad == 0);

// reset
solver.reset();
assert(solver.data == 0);
assert(solver.grad == 0);

// total length (allocated elements)
assert(solver.values.length == 4);

Tape

Create tapes of equations and update the resulting value:

// init
auto tape = new Tape();
assert(tape.values == []);
assert(tape.values.length == 0);
assert(tape.locked == false);
assert(!tape.isLocked);

// d = a * b - c
auto a = 5.value;
auto b = 10.value;
auto c = 25.value;
auto d = a * b;
auto e = d - c;
assert(e.data == 25);

// push
tape.pushBack(a);
tape ~= b;
tape ~= [c, d, e];
assert(tape.values == [a, b, c, d, e]);
assert(tape.values.length == 5);
assert(tape.lastValue.data == 25);

// lock tape
tape.lock();
// tape ~= 24.value; // assert error: reset the tape to push new values

// modify value
a.data = 6;

// update tape
tape.update();
assert(tape.lastValue.data == 35);

// reset tape to push new values
tape.reset();
tape ~= 35.value; // good

Multi-layer perceptron

import rk.tautodiff;

import std.array : array;
import std.stdio : writefln;
import std.algorithm : map;

// define data
auto input = [  // binary
    [0, 0, 0, 0], // 0
    [0, 0, 0, 1], // 1
    [0, 0, 1, 0], // 2
    [0, 0, 1, 1], // 3
    [0, 1, 0, 0], // 4
    [0, 1, 0, 1], // 5
    [0, 1, 1, 0], // 6
    [0, 1, 1, 1], // 7
    [1, 0, 0, 0], // 8
    [1, 0, 0, 1], // 9
    [1, 0, 1, 0], // 10
    [1, 0, 1, 1], // 11
    [1, 1, 0, 0], // 12
    [1, 1, 0, 1], // 13
    [1, 1, 1, 0], // 14
    [1, 1, 1, 1], // 15
].map!(x => x.map!(y => y.value).array).array;

auto target = [ // 1: even, 0: odd
    1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0
].map!(x => x.value).array;

// split train, test
auto input_train = input[0 .. 12];
auto input_test = input[12 .. $];

// define model
auto model = new MLP([4, 8, 1], &activateRelu, &activateSigmoid);

// define loss function
auto lossL2(Value[] preds)
{
    import std.algorithm : reduce;

    // voldemort type
    struct L2Loss { Value loss; float accuracy; }

    // mse loss
    Value[] losses; 
    foreach (i; 0..preds.length) losses ~= (preds[i] - target[i]) * (preds[i] - target[i]);
    auto dataLoss = losses.reduce!((a, b) => a + b) / preds.length;

    // accuracy
    float accuracy = 0.0;
    foreach (i; 0..preds.length) accuracy += ((preds[i].data > 0.5) == target[i].data);
    accuracy /= preds.length;

    // return voldemort type with cost and accuracy
    return L2Loss(dataLoss, accuracy); 
}

// train
enum lr = 0.05;
enum epochs = 100;
foreach (epoch; 0..epochs)
{
    // forward
    Value[] preds;
    foreach (x; input_train) preds ~= model.forward(x);

    // loss
    auto l2 = lossL2(preds);
    
    // backward
    model.zeroGrad();
    l2.loss.backward();
    
    // update
    model.update(lr);

    // debug print
    if (epoch % 10 == 0) writefln("epoch %3s loss %.4f accuracy %.2f", epoch, l2.loss.data, l2.accuracy);
}

// test
foreach (i, x; input_test) 
{
    auto pred = model.forward(x)[0];
    assert((pred.data > 0.5) == target[i].data);
}

Output:

epoch   0 loss 1.9461 accuracy 0.50
epoch  10 loss 0.1177 accuracy 0.75
epoch  20 loss 0.0605 accuracy 1.00
epoch  30 loss 0.0395 accuracy 1.00
...
epoch  90 loss 0.0010 accuracy 1.00

References

LICENSE

All code is licensed under the BSL license.