| import math |
| import random |
| import time |
|
|
| |
|
|
| CONTEXT_WINDOW = 5 |
| EPOCHS = 500 |
| LR = 0.01 |
|
|
| def relu(x): |
| return max(0.0, x) |
|
|
| def stable_softmax(x_list): |
| if not x_list: |
| return [] |
| m = max(x_list) |
| exps = [math.exp(i - m) for i in x_list] |
| s = sum(exps) |
| if s == 0: |
| return [1.0 / len(x_list)] * len(x_list) |
| return [e / s for e in exps] |
|
|
| class NeuralNetwork: |
| def __init__(self, layer_sizes=None, activation='relu', output_activation='softmax', |
| init_range=0.1, grad_clip=1.0, seed=None, context_window=5): |
| if seed is not None: |
| random.seed(seed) |
| self.layer_sizes = layer_sizes[:] if layer_sizes is not None else None |
| self.activation = relu if activation == 'relu' else (lambda x: x) |
| self.output_activation = stable_softmax if output_activation == 'softmax' else (lambda x: x) |
| self.init_range = float(init_range) |
| self.grad_clip = grad_clip |
| self.context_window = context_window |
| self.weights = [] |
| self.biases = [] |
| self.vocab = [] |
| self.word_to_idx = {} |
| self.idx_to_word = {} |
|
|
| def prepare_data_with_context(self, text): |
| words = [w.strip() for w in text.replace('\n', ' ').split(' ') if w.strip()] |
| self.vocab = sorted(list(set(words))) |
| self.word_to_idx = {w: i for i, w in enumerate(self.vocab)} |
| self.idx_to_word = {i: w for w, i in self.word_to_idx.items()} |
| |
| vocab_size = len(self.vocab) |
| X = [] |
| Y = [] |
| |
| for i in range(len(words) - self.context_window): |
| context_words = words[i : i + self.context_window] |
| target_word = words[i + self.context_window] |
| |
| x = [0.0] * vocab_size |
| for word in context_words: |
| if word in self.word_to_idx: |
| x[self.word_to_idx[word]] = 1.0 |
| |
| y = [0.0] * vocab_size |
| if target_word in self.word_to_idx: |
| y[self.word_to_idx[target_word]] = 1.0 |
| |
| X.append(x) |
| Y.append(y) |
| |
| return X, Y |
|
|
| def initialize_weights(self): |
| if self.layer_sizes is None: |
| raise ValueError("layer_sizes must be set before initializing weights.") |
| if self.weights: |
| return |
| for i in range(len(self.layer_sizes) - 1): |
| in_dim = self.layer_sizes[i] |
| out_dim = self.layer_sizes[i + 1] |
| W = [[random.uniform(-self.init_range, self.init_range) for _ in range(out_dim)] for _ in range(in_dim)] |
| b = [0.0 for _ in range(out_dim)] |
| self.weights.append(W) |
| self.biases.append(b) |
|
|
| def forward(self, x): |
| a = x[:] |
| for i in range(len(self.weights) - 1): |
| next_a = [] |
| W = self.weights[i] |
| b = self.biases[i] |
| out_dim = len(W[0]) |
| for j in range(out_dim): |
| s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
| next_a.append(self.activation(s)) |
| a = next_a |
|
|
| W = self.weights[-1] |
| b = self.biases[-1] |
| out = [] |
| out_dim = len(W[0]) |
| for j in range(out_dim): |
| s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
| out.append(s) |
| return self.output_activation(out) |
|
|
| def train(self, training_data, lr=0.01, epochs=500, verbose_every=50): |
| X, Y = self.prepare_data_with_context(training_data) |
| if not X: |
| raise ValueError("Not enough tokens in training data to create context windows.") |
|
|
| vocab_size = len(self.vocab) |
| if self.layer_sizes is None: |
| self.layer_sizes = [vocab_size, 64, vocab_size] |
| else: |
| self.layer_sizes[0] = vocab_size |
| self.layer_sizes[-1] = vocab_size |
|
|
| self.initialize_weights() |
|
|
| for epoch in range(epochs): |
| total_loss = 0.0 |
| indices = list(range(len(X))) |
| random.shuffle(indices) |
|
|
| for idx in indices: |
| x = X[idx] |
| y = Y[idx] |
|
|
| activations = [x[:]] |
| pre_acts = [] |
| a = x[:] |
| |
| for i in range(len(self.weights) - 1): |
| W, b = self.weights[i], self.biases[i] |
| z = [] |
| out_dim = len(W[0]) |
| for j in range(out_dim): |
| s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
| z.append(s) |
| pre_acts.append(z) |
| a = [self.activation(val) for val in z] |
| activations.append(a) |
|
|
| W, b = self.weights[-1], self.biases[-1] |
| z_final = [] |
| out_dim = len(W[0]) |
| for j in range(out_dim): |
| s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j] |
| z_final.append(s) |
| pre_acts.append(z_final) |
| out = self.output_activation(z_final) |
|
|
| delta = [out[j] - y[j] for j in range(len(y))] |
|
|
| for i in reversed(range(len(self.weights))): |
| in_act = activations[i] |
| in_dim = len(in_act) |
| out_dim = len(delta) |
| |
| db = delta[:] |
| if self.grad_clip is not None: |
| db = [max(-self.grad_clip, min(self.grad_clip, g)) for g in db] |
| for j in range(len(self.biases[i])): |
| self.biases[i][j] -= lr * db[j] |
|
|
| for k in range(in_dim): |
| for j in range(out_dim): |
| grad_w = in_act[k] * delta[j] |
| if self.grad_clip is not None: |
| grad_w = max(-self.grad_clip, min(self.grad_clip, grad_w)) |
| self.weights[i][k][j] -= lr * grad_w |
|
|
| if i != 0: |
| prev_delta = [0.0] * in_dim |
| for p in range(in_dim): |
| s = sum(self.weights[i][p][j] * delta[j] for j in range(out_dim)) |
| if pre_acts[i-1][p] > 0: |
| prev_delta[p] = s |
| delta = prev_delta |
| |
| if epoch % verbose_every == 0 or epoch == epochs - 1: |
| loss = 0.0 |
| for x_val, y_val in zip(X, Y): |
| p = self.forward(x_val) |
| for j in range(len(y_val)): |
| if y_val[j] > 0: |
| loss -= math.log(p[j] + 1e-12) |
| print(f"Epoch {epoch}, Loss: {loss / len(X):.6f}") |
|
|
| def export_to_python(self, filename): |
| lines = [] |
| lines.append("import math\n") |
| lines.append("import time\n\n") |
| lines.append("def relu(x):\n return max(0.0, x)\n\n") |
| lines.append("def softmax(x_list):\n") |
| lines.append(" if not x_list:\n") |
| lines.append(" return []\n") |
| lines.append(" m = max(x_list)\n") |
| lines.append(" exps = [math.exp(i - m) for i in x_list]\n") |
| lines.append(" s = sum(exps)\n") |
| lines.append(" if s == 0:\n") |
| lines.append(" return [1.0 / len(x_list)] * len(x_list)\n") |
| lines.append(" return [e / s for e in exps]\n\n") |
|
|
| neuron_id = 0 |
| for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)): |
| in_dim, out_dim = len(W), len(W[0]) |
| for j in range(out_dim): |
| terms = " + ".join([f"{W[i][j]:.8f}*inputs[{i}]" for i in range(in_dim)]) or "0.0" |
| b_term = f"{b[j]:.8f}" |
| if layer_idx != len(self.weights) - 1: |
| lines.append(f"def neuron_{neuron_id}(inputs):\n return relu({terms} + {b_term})\n\n") |
| else: |
| lines.append(f"def neuron_{neuron_id}(inputs):\n return {terms} + {b_term}\n\n") |
| neuron_id += 1 |
|
|
| neuron_counter = 0 |
| for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)): |
| out_dim = len(W[0]) |
| lines.append(f"def layer_{layer_idx}(inputs):\n") |
| inner = ", ".join([f"neuron_{neuron_counter + j}(inputs)" for j in range(out_dim)]) |
| lines.append(f" return [{inner}]\n\n") |
| neuron_counter += out_dim |
|
|
| lines.append("def predict(inputs):\n") |
| lines.append(" a = inputs\n") |
| for i in range(len(self.weights)): |
| lines.append(f" a = layer_{i}(a)\n") |
| lines.append(" return softmax(a)\n\n") |
|
|
| lines.append(f"vocab = {self.vocab}\n") |
| lines.append(f"word_to_idx = {{w: i for i, w in enumerate(vocab)}}\n") |
| lines.append(f"context_window = {self.context_window}\n\n") |
|
|
| lines.append("if __name__ == '__main__':\n") |
| lines.append(" print('Interactive multi-word text completion.')\n") |
| lines.append(" print(f'Model context window: {context_window} words. Type text or empty to exit.')\n") |
| lines.append(" while True:\n") |
| lines.append(" inp = input('> ').strip()\n") |
| lines.append(" if not inp:\n") |
| lines.append(" break\n") |
| lines.append(" words = [w.strip() for w in inp.split(' ') if w.strip()]\n") |
| lines.append(" generated_words = words[:]\n") |
| lines.append(" print('Input:', ' '.join(generated_words), end='', flush=True)\n") |
| lines.append(" for _ in range(20):\n") |
| lines.append(" context = generated_words[-context_window:]\n") |
| lines.append(" x = [0.0] * len(vocab)\n") |
| lines.append(" for word in context:\n") |
| lines.append(" if word in word_to_idx:\n") |
| lines.append(" x[word_to_idx[word]] = 1.0\n") |
| lines.append(" out = predict(x)\n") |
| lines.append(" idx = out.index(max(out))\n") |
| lines.append(" next_word = vocab[idx]\n") |
| lines.append(" if next_word == '<|endoftext|>': break\n") |
| lines.append(" generated_words.append(next_word)\n") |
| lines.append(" print(' ' + next_word, end='', flush=True)\n") |
| lines.append(" time.sleep(0.1)\n") |
| lines.append(" print('\\n')\n") |
|
|
| with open(filename, "w") as f: |
| f.writelines(lines) |
| print(f"Exported network to {filename}") |
|
|
| @staticmethod |
| def load_network(filename): |
| ns = {"__name__": "__loaded_model__"} |
| with open(filename, "r") as f: |
| code = f.read() |
| exec(code, ns) |
| class ModelWrapper: |
| def __init__(self, ns): |
| self.ns = ns |
| self.vocab = ns.get("vocab", []) |
| self.word_to_idx = ns.get("word_to_idx", {}) |
| self.context_window = ns.get("context_window", 5) |
|
|
| def complete(self, input_text, max_new_words=20): |
| words = [w.strip() for w in input_text.strip().split(' ') if w.strip()] |
| generated = words[:] |
| for _ in range(max_new_words): |
| context = generated[-self.context_window:] |
| x = [0.0] * len(self.vocab) |
| for word in context: |
| if word in self.word_to_idx: |
| x[self.word_to_idx[word]] = 1.0 |
| |
| out = self.ns["predict"](x) |
| idx = out.index(max(out)) |
| next_word = self.vocab[idx] |
| |
| if next_word == '<|endoftext|>': |
| break |
| generated.append(next_word) |
| return ' '.join(generated) |
|
|
| return ModelWrapper(ns) |
|
|
|
|
| if __name__ == "__main__": |
| sample_text = """ |
| user: hi |
| ai: Hello! How can I help you today? |
| <|endoftext|> |
| user: hi |
| ai: Hi! What can I do for you today? |
| <|endoftext|> |
| user: hello |
| ai: Hello! How can I help you today? |
| <|endoftext|> |
| user: hey |
| ai: Hi! What can I do for you today? |
| <|endoftext|> |
| user: How's your day going? |
| ai: It's been great! Thanks for asking! How about yours? |
| <|endoftext|> |
| user: What's new with you? |
| ai: Not much, just here and ready to help! What's new with you? |
| <|endoftext|> |
| user: What can you do? |
| ai: I can help you with a variety of tasks. What's on your mind? |
| <|endoftext|> |
| user: Tell me a joke. |
| ai: Why did the scarecrow win an award? Because he was outstanding in his field! |
| <|endoftext|> |
| """ |
| nn = NeuralNetwork(context_window=CONTEXT_WINDOW, seed=42) |
| nn.train(training_data=sample_text, lr=LR, epochs=EPOCHS, verbose_every=100) |
| nn.export_to_python("exported_model.py") |
|
|
| model = NeuralNetwork.load_network("exported_model.py") |
| print("\n--- Testing loaded model ---") |
| print(f"Vocabulary size: {len(model.vocab)}") |
| |
| test_inputs = ["user: hi", "user: What's new", "ai: It's been"] |
| for test_input in test_inputs: |
| completion = model.complete(test_input, max_new_words=10) |
| print(f"Input: '{test_input}'\nOutput: '{completion}'\n") |
