From ac0cdfdc92d6a65e79eee4e52f3b81c6fe5b4ea4 Mon Sep 17 00:00:00 2001
From: Mattia Giambirtone <nocturn9x@nocturn9x.space>
Date: Mon, 20 Mar 2023 12:11:40 +0100
Subject: [PATCH] Moved utility functions to neural network library

---
 src/main.nim       | 46 ++++---------------------------------------
 src/nn/network.nim | 49 +++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 52 insertions(+), 43 deletions(-)

diff --git a/src/main.nim b/src/main.nim
index a7cf3d7..1d2d039 100644
--- a/src/main.nim
+++ b/src/main.nim
@@ -1,49 +1,11 @@
 import nn/network
 import nn/util/matrix
 
-import std/math
 
-
-
-# Mean squared error
-proc mse(a, b: Matrix[float]): float = 
-    result = (b - a).apply(proc (x: float): float = pow(x, 2), axis = -1).sum() / len(a).float
-
-# Derivative of MSE
-func dxMSE*(x, y: Matrix[float]): Matrix[float] = 2.0 * (x - y)
-
-# A bunch of vectorized activation functions
-func sigmoid*(input: Matrix[float]): Matrix[float] = 
-    result = input.apply(proc (x: float): float = 1 / (1 + exp(-x)) , axis = -1)
-
-func sigmoidDerivative*(input: Matrix[float]): Matrix[float] = sigmoid(input) * (1.0 - sigmoid(input))
-
-
-func softmax*(input: Matrix[float]): Matrix[float] = 
-    var input = input - input.max()
-    result = input.apply(math.exp, axis = -1) / input.apply(math.exp, axis = -1).sum()
-
-
-func softmaxDerivative*(input: Matrix[float]): Matrix[float] =
-    var input = input.reshape(input.shape.cols, 1)
-    result = input.diagflat() - input.dot(input.transpose())
-
-
-func step*(input: Matrix[float]): Matrix[float] = input.apply(proc (x: float): float = (if x < 0.0: 0.0 else: x), axis = -1)
-func silu*(input: Matrix[float]): Matrix[float] = input.apply(proc (x: float): float = 1 / (1 + exp(-x)), axis= -1)
-func relu*(input: Matrix[float]): Matrix[float] = input.apply(proc (x: float): float = max(0.0, x), axis = -1)
-
-func htan*(input: Matrix[float]): Matrix[float] = 
-    let f = proc (x: float): float = 
-        let temp = exp(2 * x)
-        result = (temp - 1) / (temp + 1)
-    input.apply(f, axis = -1)
-
-
-var mlp = newNeuralNetwork(@[newDenseLayer(2, 3, newActivation(sigmoid, sigmoidDerivative)), 
-                             newDenseLayer(3, 2, newActivation(sigmoid, sigmoidDerivative)),
-                             newDenseLayer(2, 3, newActivation(softmax, softmaxDerivative))],
-                           lossFunc=newLoss(mse, dxMSE), learnRate=0.05, momentum=0.55,
+var mlp = newNeuralNetwork(@[newDenseLayer(2, 3, Sigmoid), 
+                             newDenseLayer(3, 2, Sigmoid),
+                             newDenseLayer(2, 3, Softmax)],
+                           lossFunc=MSE, learnRate=0.05, momentum=0.55,
                            weightRange=(start: -1.0, stop: 1.0), biasRange=(start: -10.0, stop: 10.0))
 echo mlp.feedforward(newMatrix[float](@[1.0, 2.0]))
 
diff --git a/src/nn/network.nim b/src/nn/network.nim
index 861b399..1c98b79 100644
--- a/src/nn/network.nim
+++ b/src/nn/network.nim
@@ -17,6 +17,7 @@ import util/matrix
 
 import std/strformat
 import std/random
+import std/math
 
 
 randomize()
@@ -142,4 +143,50 @@ proc feedforward*(self: NeuralNetwork, data: Matrix[float]): Matrix[float] =
 
 proc backprop(self: NeuralNetwork, x, y: Matrix[float]) {.used.} =
     ## Performs a single backpropagation step and updates the
-    ## gradients for our weights and biases, layer by layer
\ No newline at end of file
+    ## gradients for our weights and biases, layer by layer
+
+
+## Utility functions
+
+# Mean squared error
+proc mse(a, b: Matrix[float]): float = 
+    result = (b - a).apply(proc (x: float): float = pow(x, 2), axis = -1).sum() / len(a).float
+
+# Derivative of MSE
+func dxMSE(x, y: Matrix[float]): Matrix[float] = 2.0 * (x - y)
+
+# A bunch of vectorized activation functions
+func sigmoid(input: Matrix[float]): Matrix[float] = 
+    result = input.apply(proc (x: float): float = 1 / (1 + exp(-x)) , axis = -1)
+
+func sigmoidDerivative(input: Matrix[float]): Matrix[float] = sigmoid(input) * (1.0 - sigmoid(input))
+
+
+func softmax(input: Matrix[float]): Matrix[float] = 
+    var input = input - input.max()
+    result = input.apply(math.exp, axis = -1) / input.apply(math.exp, axis = -1).sum()
+
+
+func softmaxDerivative(input: Matrix[float]): Matrix[float] =
+    var input = input.reshape(input.shape.cols, 1)
+    result = input.diagflat() - input.dot(input.transpose())
+
+
+func step(input: Matrix[float]): Matrix[float] {.used.} = input.apply(proc (x: float): float = (if x < 0.0: 0.0 else: x), axis = -1)
+func silu(input: Matrix[float]): Matrix[float] {.used.} = input.apply(proc (x: float): float = 1 / (1 + exp(-x)), axis= -1)
+func relu(input: Matrix[float]): Matrix[float] {.used.} = input.apply(proc (x: float): float = max(0.0, x), axis = -1)
+
+func htan(input: Matrix[float]): Matrix[float] {.used.} = 
+    let f = proc (x: float): float = 
+        let temp = exp(2 * x)
+        result = (temp - 1) / (temp + 1)
+    input.apply(f, axis = -1)
+
+{.push.}
+{.hints: off.}   # So nim doesn't complain about the naming
+var Sigmoid* = newActivation(sigmoid, sigmoidDerivative)
+var Softmax* = newActivation(softmax, softmaxDerivative)
+var MSE* = newLoss(mse, dxMSE)
+{.pop.}
+
+