-
David Lanzendörfer authored20521bab
tty3.py 2.84 KiB
import json
from fac_tools import run_command
from fac_tools import get_fac_wrapper
from fac_tools import load_weights_and_biases
from fac_tools import dump_neural_network
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")
prompt = "Taxation is theft"
input_ids = tokenizer(prompt, return_tensors="pt").input_ids
tokens = input_ids[0].numpy()
print(tokens)
server = get_fac_wrapper("telnet")
INTMAX=2147483647
lr=0.95
decay_rate=10
run_command(server,"HELLO")
# Initialize the values
run_command(server,"INIT")
run_command(server,"WEIGHTS")
run_command(server,"INIT")
run_command(server,"RWEIGHTS")
run_command(server,"INIT")
run_command(server,"BIAS")
run_command(server,"TRAIN")
run_command(server,"LEARNING_RATE")
# PicoRV does not have a floating point unit
# All weights are stored as signed 32 bit integers
# this means we have to do the math here and multiply
# the maximum integer value by our desired learning rate
lr=lr*INTMAX
run_command(server,str(int(lr)))
run_command(server,"TRAIN")
run_command(server,"DECAY_RATE")
# The decay rate is a hyper parameter
# Because delta W aka alpha is calculated
# as the initial learning rate α0 multiplied
# as shown below anything floaty doesn't make sense
# anyway. Usually the decay rate is set to 1
# α=(1/(1+decayRate×epochNumber))*α0
run_command(server,str(decay_rate))
# Priming phase!
# Upload and train token pairs first
run_command(server,"TRAIN")
run_command(server,"TOKENS")
tok=tokens[0]
run_command(server,str(tok))
tok=tokens[1]
run_command(server,str(tok))
run_command(server,"DONE")
run_command(server,"TRAIN")
run_command(server,"RUN_EPOCHS")
run_command(server,str(1000))
run_command(server,"TRAIN")
run_command(server,"TOKENS")
tok=tokens[1]
run_command(server,str(tok))
tok=tokens[2]
run_command(server,str(tok))
run_command(server,"DONE")
run_command(server,"TRAIN")
run_command(server,"RUN_EPOCHS")
run_command(server,str(1000))
run_command(server,"TRAIN")
run_command(server,"TOKENS")
tok=tokens[2]
run_command(server,str(tok))
tok=tokens[3]
run_command(server,str(tok))
run_command(server,"DONE")
run_command(server,"TRAIN")
run_command(server,"RUN_EPOCHS")
run_command(server,str(1000))
# Upload token series
run_command(server,"TRAIN")
run_command(server,"TOKENS")
for tok in tokens:
run_command(server,str(tok))
run_command(server,"DONE")
run_command(server,"TRAIN")
run_command(server,"RUN_EPOCHS")
run_command(server,str(1000))
# Store the weights and biases
weights_and_biases = dump_neural_network(server)
j = json.dumps(weights_and_biases, indent=4, sort_keys=True)
print(j)
run_command(server,"TERMINATE")
server.close()
print("Writing out JSON")
with open("result/weights_and_biases_trained.json", "w") as f:
f.write(j)
f.close()