commit from alexey

config.json

@@ -1,55 +1,36 @@
 {
-    "model_path": "/mnt/10tb/home/shmelev/dnalm/downstream_tasks/APARENT/bert_base_sparse_rope_4096_bs256_lr_5e-05_wd0.01_fp16_from_425k/",
-    "log_interval": 250,
-    "valid_interval": 1000,
-    "save_interval": null,
-    "save_best": true,
-    "use_generate_on_valid": false,
-    "init_checkpoint": "/mnt/10tb/home/shmelev/dnalm/downstream_tasks/APARENT/bert_base_sparse_rope_4096_bs256_lr_5e-05_wd0.01_fp16_from_425k/model_best_from_s3.pth",
-    "skip_used_data": false,
-    "reset_lr": true,
-    "reset_iteration": true,
-    "reset_optimizer": true,
-    "lr": 5e-05,
-    "batch_size": 32,
-    "iters": 500000,
-    "gradient_accumulation_steps": 1,
-    "fp16": true,
-    "fp16_allreduce": false,
-    "apex_opt_lvl": "O2",
-    "min_loss_scale": null,
-    "max_loss_scale": 16777216,
-    "clip_grad_norm": null,
-    "clip_grad_value": null,
-    "early_stopping_patience": null,
-    "lr_scheduler": "constant_with_warmup",
-    "num_warmup_steps": 1500,
-    "num_training_steps": null,
-    "use_lr_drop": false,
-    "lr_drop_factor": 0.1,
-    "lr_drop_patience": 10,
-    "lr_drop_threshold": 0.0001,
-    "lr_drop_threshold_mode": "rel",
-    "lr_drop_cooldown": 0,
-    "lr_drop_min_lr": 0.0,
-    "lr_drop_eps": 1e-08,
-    "optimize_metric": "pearsonr2",
-    "optimize_mode": "max",
-    "train_csv": "/mnt/10tb/home/shmelev/dnalm/downstream_tasks/APARENT/dataset_itself/APARENT_train.csv",
-    "test_csv": "/mnt/10tb/home/shmelev/dnalm/downstream_tasks/APARENT/dataset_itself/APARENT_test.csv",
-    "seed": 42,
-    "input_seq_len": 256,
-    "data_n_workers": 2,
-    "model_cfg": "/mnt/10tb/home/shmelev/dnalm/data/configs/L12-H768-A12-V32k-L4096-preln-sparse-rope.json",
-    "model_cls": "src.gena_lm.modeling_bert:BertForAPARENTSequenceRegression",
-    "tokenizer": "/mnt/10tb/home/shmelev/dnalm/data/tokenizers/human/BPE_32k/",
-    "optimizer": "AdamW",
-    "weight_decay": 0.0001,
-    "ENV": {
-        "CUDA_VISIBLE_DEVICES": "0"
-    },
-    "HVD_INIT": true,
-    "HVD_SIZE": 1,
-    "MACHINE": "bio-protein",
-    "COMMIT": "2513d0c4ee891e7bcd224b9c65e88e27e0fd1892"
-}
+    "architectures": [
+      "BertForMaskedLM"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "gradient_checkpointing": false,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 4096,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 3,
+    "pre_layer_norm": true,
+    "last_layer_norm": false,
+    "position_embedding_type": "rotary",
+    "rotary_base": 10000,
+    "rotary_dim": 32,
+    "transformers_version": "4.6.0.dev0",
+    "type_vocab_size": 2,
+    "use_cache": true,
+    "vocab_size": 32000,
+    "sparse_config_cls": "deepspeed.ops.sparse_attention:BigBirdSparsityConfig",
+    "sparse_attention": {
+      "num_heads": 12,
+      "block": 64,
+      "different_layout_per_head": true,
+      "num_sliding_window_blocks": 3,
+      "num_global_blocks": 2,
+      "num_random_blocks": 3
+    }
+}