Keep track of progress after epochs + fixes
This commit is contained in:
parent
84c46caba3
commit
338732e9f9
29
src/main.nim
29
src/main.nim
|
@ -84,8 +84,8 @@ proc main =
|
||||||
"isn", "isn't", "ma", "mightn", "mightn't", "mustn", "mustn't", "needn",
|
"isn", "isn't", "ma", "mightn", "mightn't", "mustn", "mustn't", "needn",
|
||||||
"needn't", "shan", "shan't", "shouldn", "shouldn't", "wasn", "wasn't",
|
"needn't", "shan", "shan't", "shouldn", "shouldn't", "wasn", "wasn't",
|
||||||
"weren", "weren't", "won", "won't", "wouldn", "wouldn't"]
|
"weren", "weren't", "won", "won't", "wouldn", "wouldn't"]
|
||||||
const epochs = 10
|
const epochs = 100
|
||||||
const batch = 100
|
const batch = 200
|
||||||
const inputSize = 512
|
const inputSize = 512
|
||||||
let encoder = newLabelEncoder()
|
let encoder = newLabelEncoder()
|
||||||
let cleaner = newTextPreprocessor(stopwords=newMatrix(stopwords), toLower=true,
|
let cleaner = newTextPreprocessor(stopwords=newMatrix(stopwords), toLower=true,
|
||||||
|
@ -97,10 +97,10 @@ proc main =
|
||||||
newDenseLayer(16, 2)],
|
newDenseLayer(16, 2)],
|
||||||
lossFunc=MSE,
|
lossFunc=MSE,
|
||||||
activationFunc=ReLU,
|
activationFunc=ReLU,
|
||||||
learnRate=0.1,
|
learnRate=5,
|
||||||
momentum=0.3,
|
momentum=0.3,
|
||||||
weightRange=(-1.0, 1.0),
|
weightRange=(-10.0, 10.0),
|
||||||
biasRange=(-1.0, 1.0))
|
biasRange=(-10.0, 10.0))
|
||||||
echo "ProjectSydney v0.2b - Accuracy test"
|
echo "ProjectSydney v0.2b - Accuracy test"
|
||||||
echo "\nLoading dataset and testset"
|
echo "\nLoading dataset and testset"
|
||||||
let loadTime = cpuTime()
|
let loadTime = cpuTime()
|
||||||
|
@ -114,10 +114,11 @@ proc main =
|
||||||
echo &"Feature count: {len(vectorizer.getFeatureNames())}"
|
echo &"Feature count: {len(vectorizer.getFeatureNames())}"
|
||||||
echo &"Vocabulary size: {len(vectorizer.getVocabulary())}"
|
echo &"Vocabulary size: {len(vectorizer.getVocabulary())}"
|
||||||
echo &"Corpus size: {len(data.corpus)}"
|
echo &"Corpus size: {len(data.corpus)}"
|
||||||
# let yTest = encoder.transform(data.testResults)
|
let yTest = encoder.transform(data.testResults)[0]
|
||||||
# let xTest = vectorizer.transform(data.testset)
|
let xTest = vectorizer.transform(data.testset)
|
||||||
var tempData: seq[float] = newSeqOfCap[float](inputSize)
|
var tempData: seq[float] = newSeqOfCap[float](inputSize)
|
||||||
var trainData: seq[tuple[x, y: Matrix[float]]] = @[]
|
var trainData: seq[tuple[x, y: Matrix[float]]] = @[]
|
||||||
|
var testData: seq[tuple[x, y: Matrix[float]]] = @[]
|
||||||
# Pad the data to fit into the network
|
# Pad the data to fit into the network
|
||||||
for i, row in xTrain:
|
for i, row in xTrain:
|
||||||
for e in row:
|
for e in row:
|
||||||
|
@ -131,6 +132,18 @@ proc main =
|
||||||
else:
|
else:
|
||||||
trainData.add((newMatrix[float](tempData), newMatrix[float](@[0.0, 1.0])))
|
trainData.add((newMatrix[float](tempData), newMatrix[float](@[0.0, 1.0])))
|
||||||
tempData.setLen(0)
|
tempData.setLen(0)
|
||||||
|
for i, row in xTest:
|
||||||
|
for e in row:
|
||||||
|
if tempData.len() == inputSize:
|
||||||
|
break
|
||||||
|
tempData.add(e)
|
||||||
|
while tempData.len() < inputSize:
|
||||||
|
tempData.add(0.0)
|
||||||
|
if yTest[i] == 1:
|
||||||
|
testData.add((newMatrix[float](tempData), newMatrix[float](@[1.0, 0.0])))
|
||||||
|
else:
|
||||||
|
testData.add((newMatrix[float](tempData), newMatrix[float](@[0.0, 1.0])))
|
||||||
|
tempData.setLen(0)
|
||||||
echo "Classifier parameters"
|
echo "Classifier parameters"
|
||||||
echo &"\tLearn rate: {classifier.learnRate}"
|
echo &"\tLearn rate: {classifier.learnRate}"
|
||||||
echo &"\tMomentum: {classifier.momentum}"
|
echo &"\tMomentum: {classifier.momentum}"
|
||||||
|
@ -142,7 +155,7 @@ proc main =
|
||||||
echo ""
|
echo ""
|
||||||
echo &"Training neural network for {epochs} epochs with batch size of {batch}"
|
echo &"Training neural network for {epochs} epochs with batch size of {batch}"
|
||||||
let trainTime = cpuTime()
|
let trainTime = cpuTime()
|
||||||
classifier.train(epochs, batch, trainData)
|
classifier.train(epochs, batch, trainData, testData)
|
||||||
echo &"Training completed in {cpuTime() - trainTime:.2f} seconds"
|
echo &"Training completed in {cpuTime() - trainTime:.2f} seconds"
|
||||||
#[echo "\nTest parameters"
|
#[echo "\nTest parameters"
|
||||||
echo &"\tTest size: {len(data.testset)}"
|
echo &"\tTest size: {len(data.testset)}"
|
||||||
|
|
|
@ -136,10 +136,10 @@ proc feed(self: Layer, x: Matrix[float]): Matrix[float] =
|
||||||
|
|
||||||
proc fastFeedForward(self: NeuralNetwork, x: Matrix[float]): Matrix[float] {.used.} =
|
proc fastFeedForward(self: NeuralNetwork, x: Matrix[float]): Matrix[float] {.used.} =
|
||||||
## Feeds the given input through the network. The
|
## Feeds the given input through the network. The
|
||||||
## (unactivated) output from the last layer is returned
|
## (activated) output from the last layer is returned
|
||||||
result = x
|
result = x
|
||||||
for layer in self.layers:
|
for layer in self.layers:
|
||||||
result = layer.feed(result)
|
result = self.activation.function(layer.feed(result))
|
||||||
|
|
||||||
|
|
||||||
proc feedForward(self: NeuralNetwork, x: Matrix[float]): seq[Matrix[float]] =
|
proc feedForward(self: NeuralNetwork, x: Matrix[float]): seq[Matrix[float]] =
|
||||||
|
@ -246,10 +246,18 @@ proc miniBatch(self: NeuralNetwork, data: seq[tuple[x, y: Matrix[float]]]) =
|
||||||
layer.weights = (layer.weights - nudge) * newWeights
|
layer.weights = (layer.weights - nudge) * newWeights
|
||||||
|
|
||||||
|
|
||||||
proc train*(self: NeuralNetwork, epochs: int, batchSize: int, data: var seq[tuple[x, y: Matrix[float]]]) =
|
proc eval(self: NeuralNetwork, data: seq[tuple[x, y: Matrix[float]]]): float =
|
||||||
|
for sample in data:
|
||||||
|
result = result + self.loss.function(self.fastFeedForward(sample.x), sample.y)
|
||||||
|
result = result / len(data).float
|
||||||
|
|
||||||
|
|
||||||
|
proc train*(self: NeuralNetwork, epochs: int, batchSize: int, data: var seq[tuple[x, y: Matrix[float]]],
|
||||||
|
test: seq[tuple[x, y: Matrix[float]]] = @[]) =
|
||||||
## Train the network on the given data for the speficied
|
## Train the network on the given data for the speficied
|
||||||
## number of epochs using the given batch size by applying
|
## number of epochs using the given batch size by applying
|
||||||
## stochastic gradient descent
|
## stochastic gradient descent. If some test data is provided,
|
||||||
|
## training progress is printed out after each epoch
|
||||||
var batches: seq[seq[tuple[x, y: Matrix[float]]]]
|
var batches: seq[seq[tuple[x, y: Matrix[float]]]]
|
||||||
for epoch in 0..<epochs:
|
for epoch in 0..<epochs:
|
||||||
# We shuffle the data so that different epochs work
|
# We shuffle the data so that different epochs work
|
||||||
|
@ -266,6 +274,9 @@ proc train*(self: NeuralNetwork, epochs: int, batchSize: int, data: var seq[tupl
|
||||||
for batch in batches:
|
for batch in batches:
|
||||||
self.miniBatch(batch)
|
self.miniBatch(batch)
|
||||||
echo &"Completed training epoch {epoch + 1}"
|
echo &"Completed training epoch {epoch + 1}"
|
||||||
|
if len(test) > 0:
|
||||||
|
echo &"Cost: {self.eval(test)}"
|
||||||
|
|
||||||
|
|
||||||
## Utility functions
|
## Utility functions
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue