Keep track of progress after epochs + fixes

src/main.nim

@@ -84,8 +84,8 @@ proc main =
  "isn", "isn't", "ma", "mightn", "mightn't", "mustn", "mustn't", "needn", 
  "needn't", "shan", "shan't", "shouldn",  "shouldn't", "wasn", "wasn't", 
  "weren", "weren't", "won", "won't", "wouldn", "wouldn't"]
-    const epochs = 10
-    const batch = 100
+    const epochs = 100
+    const batch = 200
     const inputSize = 512
     let encoder = newLabelEncoder()
     let cleaner = newTextPreprocessor(stopwords=newMatrix(stopwords), toLower=true,
@@ -97,10 +97,10 @@ proc main =
                                         newDenseLayer(16, 2)],
                                         lossFunc=MSE,
                                         activationFunc=ReLU,
-                                        learnRate=0.1,
+                                        learnRate=5,
                                         momentum=0.3,
-                                        weightRange=(-1.0, 1.0),
-                                        biasRange=(-1.0, 1.0))
+                                        weightRange=(-10.0, 10.0),
+                                        biasRange=(-10.0, 10.0))
     echo "ProjectSydney v0.2b - Accuracy test"
     echo "\nLoading dataset and testset"
     let loadTime = cpuTime()
@@ -114,10 +114,11 @@ proc main =
     echo &"Feature count: {len(vectorizer.getFeatureNames())}"
     echo &"Vocabulary size: {len(vectorizer.getVocabulary())}"
     echo &"Corpus size: {len(data.corpus)}"
-    # let yTest = encoder.transform(data.testResults)
-    # let xTest = vectorizer.transform(data.testset)
+    let yTest = encoder.transform(data.testResults)[0]
+    let xTest = vectorizer.transform(data.testset)
     var tempData: seq[float] = newSeqOfCap[float](inputSize)
     var trainData: seq[tuple[x, y: Matrix[float]]] = @[]
+    var testData: seq[tuple[x, y: Matrix[float]]] = @[]
     # Pad the data to fit into the network
     for i, row in xTrain:
         for e in row:
@@ -131,6 +132,18 @@ proc main =
         else:
             trainData.add((newMatrix[float](tempData), newMatrix[float](@[0.0, 1.0])))
         tempData.setLen(0)
+    for i, row in xTest:
+        for e in row:
+            if tempData.len() == inputSize:
+                break
+            tempData.add(e)
+        while tempData.len() < inputSize:
+            tempData.add(0.0)
+        if yTest[i] == 1:
+            testData.add((newMatrix[float](tempData), newMatrix[float](@[1.0, 0.0])))
+        else:
+            testData.add((newMatrix[float](tempData), newMatrix[float](@[0.0, 1.0])))
+        tempData.setLen(0)
     echo "Classifier parameters"
     echo &"\tLearn rate: {classifier.learnRate}"
     echo &"\tMomentum: {classifier.momentum}"
@@ -142,7 +155,7 @@ proc main =
     echo ""
     echo &"Training neural network for {epochs} epochs with batch size of {batch}"
     let trainTime = cpuTime()
-    classifier.train(epochs, batch, trainData)
+    classifier.train(epochs, batch, trainData, testData)
     echo &"Training completed in {cpuTime() - trainTime:.2f} seconds"
     #[echo "\nTest parameters"
     echo &"\tTest size: {len(data.testset)}"

src/nn/network.nim

@@ -136,10 +136,10 @@ proc feed(self: Layer, x: Matrix[float]): Matrix[float] =
 
 proc fastFeedForward(self: NeuralNetwork, x: Matrix[float]): Matrix[float] {.used.} =
     ## Feeds the given input through the network. The
-    ## (unactivated) output from the last layer is returned
+    ## (activated) output from the last layer is returned
     result = x
     for layer in self.layers:
-        result = layer.feed(result)
+        result = self.activation.function(layer.feed(result))
 
 
 proc feedForward(self: NeuralNetwork, x: Matrix[float]): seq[Matrix[float]] =
@@ -246,10 +246,18 @@ proc miniBatch(self: NeuralNetwork, data: seq[tuple[x, y: Matrix[float]]]) =
         layer.weights = (layer.weights - nudge) * newWeights
 
 
-proc train*(self: NeuralNetwork, epochs: int, batchSize: int, data: var seq[tuple[x, y: Matrix[float]]]) =
+proc eval(self: NeuralNetwork, data: seq[tuple[x, y: Matrix[float]]]): float =
+    for sample in data:
+        result = result + self.loss.function(self.fastFeedForward(sample.x), sample.y)
+    result = result / len(data).float
+
+
+proc train*(self: NeuralNetwork, epochs: int, batchSize: int, data: var seq[tuple[x, y: Matrix[float]]],
+            test: seq[tuple[x, y: Matrix[float]]] = @[]) =
     ## Train the network on the given data for the speficied 
     ## number of epochs using the given batch size by applying
-    ## stochastic gradient descent
+    ## stochastic gradient descent. If some test data is provided,
+    ## training progress is printed out after each epoch
     var batches: seq[seq[tuple[x, y: Matrix[float]]]]
     for epoch in 0..<epochs:
         # We shuffle the data so that different epochs work
@@ -266,6 +274,9 @@ proc train*(self: NeuralNetwork, epochs: int, batchSize: int, data: var seq[tupl
         for batch in batches:
             self.miniBatch(batch)
         echo &"Completed training epoch {epoch + 1}"
+        if len(test) > 0:
+            echo &"Cost: {self.eval(test)}"
+
 
 ## Utility functions