TinyStories-24m: 24.59M BPE GPT trained from scratch on TinyStories (val ppl 8.76, coherent)
c6c9cdf verified | { | |
| "architectures": ["TinyStoriesGPT"], | |
| "model_type": "tinystories-gpt", | |
| "D": 384, | |
| "L": 12, | |
| "H": 8, | |
| "FFN": 1536, | |
| "vocab_size": 8192, | |
| "max_position_embeddings": 512, | |
| "n_params": 24585600, | |
| "tie_word_embeddings": true, | |
| "norm": "RMSNorm", | |
| "ffn": "GELU", | |
| "attn": "causal SDPA (fused qkv)", | |
| "trained_on": "roneneldan/TinyStories", | |
| "step": 6000, | |
| "val_nats": 2.1618, | |
| "val_ppl": 8.76, | |
| "dtype": "float32" | |
| } | |