From 0d70526d82bbacd5bf3a82ebe1f8b0e6ec694e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Lanzend=C3=B6rfer?= <leviathan@libresilicon.com>
Date: Sun, 30 Jun 2024 08:53:15 +0100
Subject: [PATCH] Update training script
---
Makefile | 2 +-
firmware/rnn.c | 12 ++++-----
src/cpptb/layer.cpp | 2 +-
src/cpptb/network.cpp | 2 +-
src/py/tty3.py | 20 +++++++++------
src/rtl/layer.sv | 2 +-
src/rtl/network.v | 50 ++++++++++++++++--------------------
src/rtl/network_controller.v | 4 ++-
8 files changed, 47 insertions(+), 47 deletions(-)
diff --git a/Makefile b/Makefile
index 6e53018..3ebad1a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
# Parameters of network:
NETWORK_PARAMS = \
- NUM_BITS_PER_TOKEN:16 \
+ NUM_INPUT_SYNAPSES:16 \
NUM_INPUT_NEURONS:5 \
NUM_OUTPUT_NEURONS:16 \
NUM_HIDDEN_NEURONS_W:5 \
diff --git a/firmware/rnn.c b/firmware/rnn.c
index 0633a49..05c7b69 100644
--- a/firmware/rnn.c
+++ b/firmware/rnn.c
@@ -4,7 +4,7 @@
* Acces functions for the encoder layer
*/
-#define ENCODER_MEM_SIZE (NUM_BITS_PER_TOKEN+NUM_INPUT_NEURONS+2)
+#define ENCODER_MEM_SIZE (NUM_INPUT_SYNAPSES+NUM_INPUT_NEURONS+2)
#define HIDDEN_MEM_SIZE (NUM_INPUT_NEURONS+NUM_HIDDEN_NEURONS_H+2)
#define DECODER_MEM_SIZE (NUM_HIDDEN_NEURONS_H+NUM_OUTPUT_NEURONS+2)
@@ -53,7 +53,7 @@ uint32_t get_layer_rnn_offset(enum layer_type layer)
uint32_t ret = 0;
switch(layer) {
case LAYER_TYPE_ENCODER:
- ret = NUM_BITS_PER_TOKEN;
+ ret = NUM_INPUT_SYNAPSES;
break;
case LAYER_TYPE_HIDDEN:
ret = NUM_INPUT_NEURONS;
@@ -70,7 +70,7 @@ uint32_t get_layer_bias_offset(enum layer_type layer)
uint32_t ret = 0;
switch(layer) {
case LAYER_TYPE_ENCODER:
- ret = NUM_BITS_PER_TOKEN+NUM_INPUT_NEURONS;
+ ret = NUM_INPUT_SYNAPSES+NUM_INPUT_NEURONS;
break;
case LAYER_TYPE_HIDDEN:
ret = NUM_INPUT_NEURONS+NUM_HIDDEN_NEURONS_H;
@@ -87,7 +87,7 @@ uint32_t get_layer_dw_offset(enum layer_type layer)
uint32_t ret = 0;
switch(layer) {
case LAYER_TYPE_ENCODER:
- ret = NUM_BITS_PER_TOKEN+NUM_INPUT_NEURONS+1;
+ ret = NUM_INPUT_SYNAPSES+NUM_INPUT_NEURONS+1;
break;
case LAYER_TYPE_HIDDEN:
ret = NUM_INPUT_NEURONS+NUM_HIDDEN_NEURONS_H+1;
@@ -181,7 +181,7 @@ uint32_t get_layer_weight_count(enum layer_type layer)
uint32_t ret = 0;
switch(layer) {
case LAYER_TYPE_ENCODER:
- ret = NUM_BITS_PER_TOKEN;
+ ret = NUM_INPUT_SYNAPSES;
break;
case LAYER_TYPE_HIDDEN:
ret = NUM_INPUT_NEURONS;
@@ -343,7 +343,7 @@ void init_random_weights()
{
int i, j, x, y;
for(i=0;i<NUM_INPUT_NEURONS;i++)
- for(j=0;j<NUM_BITS_PER_TOKEN;j++)
+ for(j=0;j<NUM_INPUT_SYNAPSES;j++)
set_layer_weight(LAYER_TYPE_ENCODER,LAYER_VALUE_TYPE_WEIGHTS,i, j, get_random_value());
for(i=0;i<NUM_HIDDEN_NEURONS_W*NUM_HIDDEN_NEURONS_H;i++)
for(j=0;j<NUM_INPUT_NEURONS;j++)
diff --git a/src/cpptb/layer.cpp b/src/cpptb/layer.cpp
index a9e4db8..e16809c 100644
--- a/src/cpptb/layer.cpp
+++ b/src/cpptb/layer.cpp
@@ -19,7 +19,7 @@
* and one additional byte for the bias
*
*/
-#define MEM_SIZE_NEURON (NUM_BITS_PER_TOKEN+TOTAL_NEURONS+1)
+#define MEM_SIZE_NEURON (NUM_INPUT_SYNAPSES+TOTAL_NEURONS+1)
/* Total memory is the amount of neurons multiplied
* by the memory of one neuron (perceptron)
diff --git a/src/cpptb/network.cpp b/src/cpptb/network.cpp
index 560c373..53ce0d4 100644
--- a/src/cpptb/network.cpp
+++ b/src/cpptb/network.cpp
@@ -19,7 +19,7 @@
* and one additional byte for the bias
*
*/
-#define MEM_SIZE_NEURON (NUM_BITS_PER_TOKEN+TOTAL_NEURONS+1)
+#define MEM_SIZE_NEURON (NUM_INPUT_SYNAPSES+TOTAL_NEURONS+1)
/* Total memory is the amount of neurons multiplied
* by the memory of one neuron (perceptron)
diff --git a/src/py/tty3.py b/src/py/tty3.py
index e05a6ab..c76585f 100644
--- a/src/py/tty3.py
+++ b/src/py/tty3.py
@@ -15,19 +15,23 @@ print(tokens)
server = get_fac_wrapper("telnet")
-print("Reading in JSON")
-with open("test_files/weights_and_biases.json", "r") as f:
- weights_and_biases = json.loads(f.read())
- f.close()
-
-load_weights_and_biases(server, weights_and_biases)
-
INTMAX=2147483647
lr=0.95
decay_rate=1
run_command(server,"HELLO")
+# Initialize the values
+run_command(server,"INIT")
+run_command(server,"WEIGHTS")
+
+run_command(server,"INIT")
+run_command(server,"RWEIGHTS")
+
+run_command(server,"INIT")
+run_command(server,"BIAS")
+
+# Upload token series
run_command(server,"TRAIN")
run_command(server,"TOKENS")
for tok in tokens:
@@ -58,8 +62,8 @@ run_command(server,"TRAIN")
run_command(server,"RUN_EPOCHS")
run_command(server,str(20000))
+# Store the weights and biases
weights_and_biases = dump_neural_network(server)
-
j = json.dumps(weights_and_biases, indent=4, sort_keys=True)
print(j)
diff --git a/src/rtl/layer.sv b/src/rtl/layer.sv
index ba428d3..647f91a 100644
--- a/src/rtl/layer.sv
+++ b/src/rtl/layer.sv
@@ -1,5 +1,5 @@
module layer #(
- parameter NUMBER_SYNAPSES = `NUM_BITS_PER_TOKEN,
+ parameter NUMBER_SYNAPSES = `NUM_INPUT_SYNAPSES,
parameter NUMBER_NEURONS = `NUM_INPUT_NEURONS
)
(
diff --git a/src/rtl/network.v b/src/rtl/network.v
index 4103a90..cb88f25 100644
--- a/src/rtl/network.v
+++ b/src/rtl/network.v
@@ -1,13 +1,6 @@
`include "params.vh"
-module network #(
- parameter NUMBER_SYNAPSES = `NUM_BITS_PER_TOKEN,
- parameter NUMBER_INPUT_NEURONS = `NUM_INPUT_NEURONS,
- parameter NUMBER_OUTPUT_NEURONS = `NUM_OUTPUT_NEURONS,
- parameter NUMBER_HIDDEN_NEURONS_W = `NUM_HIDDEN_NEURONS_W,
- parameter NUMBER_HIDDEN_NEURONS_H = `NUM_HIDDEN_NEURONS_H,
- parameter NUM_BITS = 32
-)
+module network
(
clk,
rst,
@@ -27,8 +20,8 @@ module network #(
wire rst_ann;
// The layer i/o
- wire[NUMBER_SYNAPSES-1:0] layer_inputs;
- wire[NUMBER_OUTPUT_NEURONS-1:0] layer_outputs;
+ wire[`NUM_INPUT_SYNAPSES-1:0] layer_inputs;
+ wire[`NUM_OUTPUT_NEURONS-1:0] layer_outputs;
// DMA interface
input mem_enable;
@@ -39,7 +32,7 @@ module network #(
output mem_ready;
// Encoder
- wire[NUMBER_INPUT_NEURONS-1:0] encoder_outputs;
+ wire[`NUM_INPUT_NEURONS-1:0] encoder_outputs;
wire [31:0] encoder_addr;
wire [31:0] encoder_data_i;
wire [31:0] encoder_data_o;
@@ -61,7 +54,7 @@ module network #(
wire hidden_read_done;
wire hidden_write_done;
- wire[NUMBER_HIDDEN_NEURONS_H-1:0] hidden_outputs;
+ wire[`NUM_HIDDEN_NEURONS_H-1:0] hidden_outputs;
// Decoder
wire [31:0] decoder_addr;
@@ -75,18 +68,18 @@ module network #(
wire decoder_write_done;
// Training interface
- wire[NUMBER_OUTPUT_NEURONS-1:0] backprop_port; // tell the other neurons that they have to better themselves
- wire[NUMBER_HIDDEN_NEURONS_H-1:0] hidden_backprop_port; // tell the other neurons that they have to better themselves
- wire[NUMBER_INPUT_NEURONS-1:0] encoder_backprop_port; // tell the other neurons that they have to better themselves
+ wire[`NUM_OUTPUT_NEURONS-1:0] backprop_port; // tell the other neurons that they have to better themselves
+ wire[`NUM_HIDDEN_NEURONS_H-1:0] hidden_backprop_port; // tell the other neurons that they have to better themselves
+ wire[`NUM_INPUT_NEURONS-1:0] encoder_backprop_port; // tell the other neurons that they have to better themselves
wire backprop_enable;
network_controller #(
- .NUMBER_SYNAPSES(NUMBER_SYNAPSES),
- .NUMBER_INPUT_NEURONS(NUMBER_INPUT_NEURONS),
- .NUMBER_OUTPUT_NEURONS(NUMBER_OUTPUT_NEURONS),
- .NUMBER_HIDDEN_NEURONS_W(NUMBER_HIDDEN_NEURONS_W),
- .NUMBER_HIDDEN_NEURONS_H(NUMBER_HIDDEN_NEURONS_H)
+ .NUMBER_SYNAPSES(`NUM_INPUT_SYNAPSES),
+ .NUMBER_INPUT_NEURONS(`NUM_INPUT_NEURONS),
+ .NUMBER_OUTPUT_NEURONS(`NUM_OUTPUT_NEURONS),
+ .NUMBER_HIDDEN_NEURONS_W(`NUM_HIDDEN_NEURONS_W),
+ .NUMBER_HIDDEN_NEURONS_H(`NUM_HIDDEN_NEURONS_H)
) controller (
.clk(clk),
.rst(rst),
@@ -104,6 +97,7 @@ module network #(
.mem_ready(mem_ready),
// Layer I/O
.encoder_input(layer_inputs),
+ .encoder_output(encoder_outputs),
.decoder_output(layer_outputs),
// The training interface
.backprop_port(backprop_port),
@@ -137,8 +131,8 @@ module network #(
// Encoder
layer #(
- .NUMBER_SYNAPSES(NUMBER_SYNAPSES),
- .NUMBER_NEURONS(NUMBER_INPUT_NEURONS)
+ .NUMBER_SYNAPSES(`NUM_INPUT_SYNAPSES),
+ .NUMBER_NEURONS(`NUM_INPUT_NEURONS)
)
encoder(
.clk(clk),
@@ -164,10 +158,10 @@ module network #(
// Hidden Layers
- hidden_network #(
- .NUMBER_SYNAPSES(NUMBER_INPUT_NEURONS),
- .NUMBER_NEURONS_W(NUMBER_HIDDEN_NEURONS_W),
- .NUMBER_NEURONS_H(NUMBER_HIDDEN_NEURONS_H)
+ neuron_matrix #(
+ .NUMBER_SYNAPSES(`NUM_INPUT_NEURONS),
+ .NUMBER_NEURONS_W(`NUM_HIDDEN_NEURONS_W),
+ .NUMBER_NEURONS_H(`NUM_HIDDEN_NEURONS_H)
)
hidden_layers(
.clk(clk),
@@ -193,8 +187,8 @@ module network #(
// Decoder
layer #(
- .NUMBER_SYNAPSES(NUMBER_HIDDEN_NEURONS_H),
- .NUMBER_NEURONS(NUMBER_OUTPUT_NEURONS)
+ .NUMBER_SYNAPSES(`NUM_HIDDEN_NEURONS_H),
+ .NUMBER_NEURONS(`NUM_OUTPUT_NEURONS)
)
decoder(
.clk(clk),
diff --git a/src/rtl/network_controller.v b/src/rtl/network_controller.v
index 36e85fc..a2e3810 100644
--- a/src/rtl/network_controller.v
+++ b/src/rtl/network_controller.v
@@ -23,7 +23,7 @@
`define TRAINING_SINGLE_SHOT 1
module network_controller #(
- parameter NUMBER_SYNAPSES = `NUM_BITS_PER_TOKEN,
+ parameter NUMBER_SYNAPSES = `NUM_INPUT_SYNAPSES,
parameter NUMBER_INPUT_NEURONS = `NUM_INPUT_NEURONS,
parameter NUMBER_OUTPUT_NEURONS = `NUM_OUTPUT_NEURONS,
parameter NUMBER_HIDDEN_NEURONS_W = `NUM_HIDDEN_NEURONS_W,
@@ -46,6 +46,7 @@ module network_controller #(
mem_ready,
// The layer i/o
encoder_input,
+ encoder_output,
decoder_output,
// The backprop port
backprop_port,
@@ -96,6 +97,7 @@ module network_controller #(
// The layer i/o
output reg[NUMBER_SYNAPSES-1:0] encoder_input;
+ input[NUMBER_INPUT_NEURONS-1:0] encoder_output;
input[NUMBER_OUTPUT_NEURONS-1:0] decoder_output;
// Training
--
GitLab