|
|
import math |
|
|
import random |
|
|
import time |
|
|
|
|
|
|
|
|
|
|
|
CONTEXT_WINDOW = 5 |
|
|
EPOCHS = 500 |
|
|
LR = 0.01 |
|
|
|
|
|
def relu(x): |
|
|
return max(0.0, x) |
|
|
|
|
|
def stable_softmax(x_list): |
|
|
if not x_list: |
|
|
return [] |
|
|
m = max(x_list) |
|
|
exps = [math.exp(i - m) for i in x_list] |
|
|
s = sum(exps) |
|
|
if s == 0: |
|
|
return [1.0 / len(x_list)] * len(x_list) |
|
|
return [e / s for e in exps] |
|
|
|
|
|
class NeuralNetwork: |
|
|
def __init__(self, layer_sizes=None, activation='relu', output_activation='softmax', |
|
|
init_range=0.1, grad_clip=1.0, seed=None, context_window=5): |
|
|
if seed is not None: |
|
|
random.seed(seed) |
|
|
self.layer_sizes = layer_sizes[:] if layer_sizes is not None else None |
|
|
self.activation = relu if activation == 'relu' else (lambda x: x) |
|
|
self.output_activation = stable_softmax if output_activation == 'softmax' else (lambda x: x) |
|
|
self.init_range = float(init_range) |
|
|
self.grad_clip = grad_clip |
|
|
self.context_window = context_window |
|
|
self.weights = [] |
|
|
self.biases = [] |
|
|
self.vocab = [] |
|
|
self.word_to_idx = {} |
|
|
self.idx_to_word = {} |
|
|
|
|
|
def prepare_data_with_context(self, text): |
|
|
words = [w.strip() for w in text.replace('\n', ' ').split(' ') if w.strip()] |
|
|
self.vocab = sorted(list(set(words))) |
|
|
self.word_to_idx = {w: i for i, w in enumerate(self.vocab)} |
|
|
self.idx_to_word = {i: w for w, i in self.word_to_idx.items()} |
|
|
|
|
|
vocab_size = len(self.vocab) |
|
|
X = [] |
|
|
Y = [] |
|
|
|
|
|
for i in range(len(words) - self.context_window): |
|
|
context_words = words[i : i + self.context_window] |
|
|
target_word = words[i + self.context_window] |
|
|
|
|
|
x = [0.0] * vocab_size |
|
|
for word in context_words: |
|
|
if word in self.word_to_idx: |
|
|
x[self.word_to_idx[word]] = 1.0 |
|
|
|
|
|
y = [0.0] * vocab_size |
|
|
if target_word in self.word_to_idx: |
|
|
y[self.word_to_idx[target_word]] = 1.0 |
|
|
|
|
|
X.append(x) |
|
|
Y.append(y) |
|
|
|
|
|
return X, Y |
|
|
|
|
|
def initialize_weights(self): |
|
|
if self.layer_sizes is None: |
|
|
raise ValueError("layer_sizes must be set before initializing weights.") |
|
|
if self.weights: |
|
|
return |
|
|
for i in range(len(self.layer_sizes) - 1): |
|
|
in_dim = self.layer_sizes[i] |
|
|
out_dim = self.layer_sizes[i + 1] |
|
|
W = [[random.uniform(-self.init_range, self.init_range) for _ in range(out_dim)] for _ in range(in_dim)] |
|
|
b = [0.0 for _ in range(out_dim)] |
|
|
self.weights.append(W) |
|
|
self.biases.append(b) |
|
|
|
|
|
def forward(self, x): |
|
|
a = x[:] |
|
|
for i in range(len(self.weights) - 1): |
|
|
next_a = [] |
|
|
W = self.weights[i] |
|
|
b = self.biases[i] |
|
|
out_dim = len(W[0]) |
|
|
for j in range(out_dim): |
|
|
s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
|
|
next_a.append(self.activation(s)) |
|
|
a = next_a |
|
|
|
|
|
W = self.weights[-1] |
|
|
b = self.biases[-1] |
|
|
out = [] |
|
|
out_dim = len(W[0]) |
|
|
for j in range(out_dim): |
|
|
s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
|
|
out.append(s) |
|
|
return self.output_activation(out) |
|
|
|
|
|
def train(self, training_data, lr=0.01, epochs=500, verbose_every=50): |
|
|
X, Y = self.prepare_data_with_context(training_data) |
|
|
if not X: |
|
|
raise ValueError("Not enough tokens in training data to create context windows.") |
|
|
|
|
|
vocab_size = len(self.vocab) |
|
|
if self.layer_sizes is None: |
|
|
self.layer_sizes = [vocab_size, 64, vocab_size] |
|
|
else: |
|
|
self.layer_sizes[0] = vocab_size |
|
|
self.layer_sizes[-1] = vocab_size |
|
|
|
|
|
self.initialize_weights() |
|
|
|
|
|
for epoch in range(epochs): |
|
|
total_loss = 0.0 |
|
|
indices = list(range(len(X))) |
|
|
random.shuffle(indices) |
|
|
|
|
|
for idx in indices: |
|
|
x = X[idx] |
|
|
y = Y[idx] |
|
|
|
|
|
activations = [x[:]] |
|
|
pre_acts = [] |
|
|
a = x[:] |
|
|
|
|
|
for i in range(len(self.weights) - 1): |
|
|
W, b = self.weights[i], self.biases[i] |
|
|
z = [] |
|
|
out_dim = len(W[0]) |
|
|
for j in range(out_dim): |
|
|
s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
|
|
z.append(s) |
|
|
pre_acts.append(z) |
|
|
a = [self.activation(val) for val in z] |
|
|
activations.append(a) |
|
|
|
|
|
W, b = self.weights[-1], self.biases[-1] |
|
|
z_final = [] |
|
|
out_dim = len(W[0]) |
|
|
for j in range(out_dim): |
|
|
s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
|
|
z_final.append(s) |
|
|
pre_acts.append(z_final) |
|
|
out = self.output_activation(z_final) |
|
|
|
|
|
delta = [out[j] - y[j] for j in range(len(y))] |
|
|
|
|
|
for i in reversed(range(len(self.weights))): |
|
|
in_act = activations[i] |
|
|
in_dim = len(in_act) |
|
|
out_dim = len(delta) |
|
|
|
|
|
db = delta[:] |
|
|
if self.grad_clip is not None: |
|
|
db = [max(-self.grad_clip, min(self.grad_clip, g)) for g in db] |
|
|
for j in range(len(self.biases[i])): |
|
|
self.biases[i][j] -= lr * db[j] |
|
|
|
|
|
for k in range(in_dim): |
|
|
for j in range(out_dim): |
|
|
grad_w = in_act[k] * delta[j] |
|
|
if self.grad_clip is not None: |
|
|
grad_w = max(-self.grad_clip, min(self.grad_clip, grad_w)) |
|
|
self.weights[i][k][j] -= lr * grad_w |
|
|
|
|
|
if i != 0: |
|
|
prev_delta = [0.0] * in_dim |
|
|
for p in range(in_dim): |
|
|
s = sum(self.weights[i][p][j] * delta[j] for j in range(out_dim)) |
|
|
if pre_acts[i-1][p] > 0: |
|
|
prev_delta[p] = s |
|
|
delta = prev_delta |
|
|
|
|
|
if epoch % verbose_every == 0 or epoch == epochs - 1: |
|
|
loss = 0.0 |
|
|
for x_val, y_val in zip(X, Y): |
|
|
p = self.forward(x_val) |
|
|
for j in range(len(y_val)): |
|
|
if y_val[j] > 0: |
|
|
loss -= math.log(p[j] + 1e-12) |
|
|
print(f"Epoch {epoch}, Loss: {loss / len(X):.6f}") |
|
|
|
|
|
def export_to_python(self, filename): |
|
|
lines = [] |
|
|
lines.append("import math\n") |
|
|
lines.append("import time\n\n") |
|
|
lines.append("def relu(x):\n return max(0.0, x)\n\n") |
|
|
lines.append("def softmax(x_list):\n") |
|
|
lines.append(" if not x_list:\n") |
|
|
lines.append(" return []\n") |
|
|
lines.append(" m = max(x_list)\n") |
|
|
lines.append(" exps = [math.exp(i - m) for i in x_list]\n") |
|
|
lines.append(" s = sum(exps)\n") |
|
|
lines.append(" if s == 0:\n") |
|
|
lines.append(" return [1.0 / len(x_list)] * len(x_list)\n") |
|
|
lines.append(" return [e / s for e in exps]\n\n") |
|
|
|
|
|
neuron_id = 0 |
|
|
for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)): |
|
|
in_dim, out_dim = len(W), len(W[0]) |
|
|
for j in range(out_dim): |
|
|
terms = " + ".join([f"{W[i][j]:.8f}*inputs[{i}]" for i in range(in_dim)]) or "0.0" |
|
|
b_term = f"{b[j]:.8f}" |
|
|
if layer_idx != len(self.weights) - 1: |
|
|
lines.append(f"def neuron_{neuron_id}(inputs):\n return relu({terms} + {b_term})\n\n") |
|
|
else: |
|
|
lines.append(f"def neuron_{neuron_id}(inputs):\n return {terms} + {b_term}\n\n") |
|
|
neuron_id += 1 |
|
|
|
|
|
neuron_counter = 0 |
|
|
for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)): |
|
|
out_dim = len(W[0]) |
|
|
lines.append(f"def layer_{layer_idx}(inputs):\n") |
|
|
inner = ", ".join([f"neuron_{neuron_counter + j}(inputs)" for j in range(out_dim)]) |
|
|
lines.append(f" return [{inner}]\n\n") |
|
|
neuron_counter += out_dim |
|
|
|
|
|
lines.append("def predict(inputs):\n") |
|
|
lines.append(" a = inputs\n") |
|
|
for i in range(len(self.weights)): |
|
|
lines.append(f" a = layer_{i}(a)\n") |
|
|
lines.append(" return softmax(a)\n\n") |
|
|
|
|
|
lines.append(f"vocab = {self.vocab}\n") |
|
|
lines.append(f"word_to_idx = {{w: i for i, w in enumerate(vocab)}}\n") |
|
|
lines.append(f"context_window = {self.context_window}\n\n") |
|
|
|
|
|
lines.append("if __name__ == '__main__':\n") |
|
|
lines.append(" print('Interactive multi-word text completion.')\n") |
|
|
lines.append(" print(f'Model context window: {context_window} words. Type text or empty to exit.')\n") |
|
|
lines.append(" while True:\n") |
|
|
lines.append(" inp = input('> ').strip()\n") |
|
|
lines.append(" if not inp:\n") |
|
|
lines.append(" break\n") |
|
|
lines.append(" words = [w.strip() for w in inp.split(' ') if w.strip()]\n") |
|
|
lines.append(" generated_words = words[:]\n") |
|
|
lines.append(" print('Input:', ' '.join(generated_words), end='', flush=True)\n") |
|
|
lines.append(" for _ in range(20):\n") |
|
|
lines.append(" context = generated_words[-context_window:]\n") |
|
|
lines.append(" x = [0.0] * len(vocab)\n") |
|
|
lines.append(" for word in context:\n") |
|
|
lines.append(" if word in word_to_idx:\n") |
|
|
lines.append(" x[word_to_idx[word]] = 1.0\n") |
|
|
lines.append(" out = predict(x)\n") |
|
|
lines.append(" idx = out.index(max(out))\n") |
|
|
lines.append(" next_word = vocab[idx]\n") |
|
|
lines.append(" if next_word == '<|endoftext|>': break\n") |
|
|
lines.append(" generated_words.append(next_word)\n") |
|
|
lines.append(" print(' ' + next_word, end='', flush=True)\n") |
|
|
lines.append(" time.sleep(0.1)\n") |
|
|
lines.append(" print('\\n')\n") |
|
|
|
|
|
with open(filename, "w") as f: |
|
|
f.writelines(lines) |
|
|
print(f"Exported network to {filename}") |
|
|
|
|
|
@staticmethod |
|
|
def load_network(filename): |
|
|
ns = {"__name__": "__loaded_model__"} |
|
|
with open(filename, "r") as f: |
|
|
code = f.read() |
|
|
exec(code, ns) |
|
|
class ModelWrapper: |
|
|
def __init__(self, ns): |
|
|
self.ns = ns |
|
|
self.vocab = ns.get("vocab", []) |
|
|
self.word_to_idx = ns.get("word_to_idx", {}) |
|
|
self.context_window = ns.get("context_window", 5) |
|
|
|
|
|
def complete(self, input_text, max_new_words=20): |
|
|
words = [w.strip() for w in input_text.strip().split(' ') if w.strip()] |
|
|
generated = words[:] |
|
|
for _ in range(max_new_words): |
|
|
context = generated[-self.context_window:] |
|
|
x = [0.0] * len(self.vocab) |
|
|
for word in context: |
|
|
if word in self.word_to_idx: |
|
|
x[self.word_to_idx[word]] = 1.0 |
|
|
|
|
|
out = self.ns["predict"](x) |
|
|
idx = out.index(max(out)) |
|
|
next_word = self.vocab[idx] |
|
|
|
|
|
if next_word == '<|endoftext|>': |
|
|
break |
|
|
generated.append(next_word) |
|
|
return ' '.join(generated) |
|
|
|
|
|
return ModelWrapper(ns) |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
sample_text = """ |
|
|
user: hi |
|
|
ai: Hello! How can I help you today? |
|
|
<|endoftext|> |
|
|
user: hi |
|
|
ai: Hi! What can I do for you today? |
|
|
<|endoftext|> |
|
|
user: hello |
|
|
ai: Hello! How can I help you today? |
|
|
<|endoftext|> |
|
|
user: hey |
|
|
ai: Hi! What can I do for you today? |
|
|
<|endoftext|> |
|
|
user: How's your day going? |
|
|
ai: It's been great! Thanks for asking! How about yours? |
|
|
<|endoftext|> |
|
|
user: What's new with you? |
|
|
ai: Not much, just here and ready to help! What's new with you? |
|
|
<|endoftext|> |
|
|
user: What can you do? |
|
|
ai: I can help you with a variety of tasks. What's on your mind? |
|
|
<|endoftext|> |
|
|
user: Tell me a joke. |
|
|
ai: Why did the scarecrow win an award? Because he was outstanding in his field! |
|
|
<|endoftext|> |
|
|
""" |
|
|
nn = NeuralNetwork(context_window=CONTEXT_WINDOW, seed=42) |
|
|
nn.train(training_data=sample_text, lr=LR, epochs=EPOCHS, verbose_every=100) |
|
|
nn.export_to_python("exported_model.py") |
|
|
|
|
|
model = NeuralNetwork.load_network("exported_model.py") |
|
|
print("\n--- Testing loaded model ---") |
|
|
print(f"Vocabulary size: {len(model.vocab)}") |
|
|
|
|
|
test_inputs = ["user: hi", "user: What's new", "ai: It's been"] |
|
|
for test_input in test_inputs: |
|
|
completion = model.complete(test_input, max_new_words=10) |
|
|
print(f"Input: '{test_input}'\nOutput: '{completion}'\n") |
