| {"sae": {"activation": "topk", "expansion_factor": 16, "normalize_decoder": true, "num_latents": 0, "k": 1539, "multi_topk": false, "skip_connection": true, "transcode": true, "optimized_encoder_config": "None_", "pkm_config": {"pad": false, "softmax": false, "heads": 1, "bias": false, "init_scale": 1.0}, "kronecker_config": {"in_group": 2, "out_group": 4, "u": 4, "lora_dim": 1.0}}, "batch_size": 32, "grad_acc_steps": 1, "micro_acc_steps": 1, "stop_steps": 5000, "loss_fn": "fvu", "optimizer": "muon", "lr": 0.0008, "lr_warmup_steps": 1000, "k_decay_steps": 4000, "auxk_alpha": 0.0, "dead_feature_threshold": 10000000, "hookpoints": ["layers.10.mlp", "layers.15.mlp", "layers.20.mlp"], "init_seeds": [0], "layers": [], "layer_stride": 1, "distribute_modules": false, "save_every": 1000, "log_to_wandb": true, "run_name": "baseline-16x-k128", "wandb_log_frequency": 1, "wandb_project": "sparsify", "model": "HuggingFaceTB/SmolLM2-135M", "dataset": "EleutherAI/fineweb-edu-dedup-10b", "split": "train", "ctx_len": 2048, "hf_token": null, "revision": null, "load_in_8bit": false, "max_examples": null, "resume": false, "text_column": "text", "finetune": null, "shuffle_seed": 42, "data_preprocessing_num_proc": 48} |