Initial commit

Co-authored-by: tsbpp <tsbpp@users.noreply.huggingface.co>
Co-authored-by: meta-bot <meta-bot@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+webssl_teaser.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,51 @@
+---
+library_name: transformers
+license: cc-by-nc-4.0
+inference: false
+---
+# Web-SSL DINO ViT-1B: 2B MetaCLIP data, 224 Resolution
+A 1 billion parameter Vision Transformer (ViT) trained with DINOv2 self-supervised learning on web-scale image data without language supervision. Introduced in ["Scaling Language-Free Visual Representation Learning"](https://arxiv.org/abs/2504.01017) (Fan et al., 2025).
+
+## Model Details
+- **Architecture**: ViT (1536 width, 40 depth, 24 heads)
+- **Parameters**: 1B
+- **Resolution**: 224×224 pixels
+- **Training**: Self-supervised Web-DINO on 2B image samples from MetaCLIP web data
+
+## Model Descriptions
+Web-SSL DINO 1B is a 1 billion parameter Vision Transformer model trained using self-supervised learning on 2 billion web images without language supervision. This model demonstrates that pure visual learning, when scaled appropriately, can match or exceed the performance of language-supervised models like CLIP across various vision tasks. 
+
+<img src="webssl_teaser.png" alt="WebSSL Model Overview" width="600">
+
+## Usage
+```python
+from transformers import AutoImageProcessor, Dinov2Model
+import torch
+from PIL import Image
+
+processor = AutoImageProcessor.from_pretrained('facebook/webssl-dino1b-full2b-224')
+# 'eager' and 'sdpa' attn_implementation supported
+model = Dinov2Model.from_pretrained('facebook/webssl-dino1b-full2b-224')
+
+# Process an image
+image = Image.open('path/to/image.jpg')
+inputs = processor(images=image, return_tensors="pt")
+with torch.no_grad():
+    outputs = model(**inputs)
+
+cls_features = outputs.last_hidden_state[:, 0]  # CLS token features
+patch_features = outputs.last_hidden_state[:, 1:] # patch-wise token features
+```
+
+## Citation
+
+```bibtex
+@article{fan2025scaling,
+  title={Scaling Language-Free Visual Representation Learning}, 
+  author={David Fan and Shengbang Tong and Jiachen Zhu and Koustuv Sinha and Zhuang Liu and Xinlei Chen and Michael Rabbat and Nicolas Ballas and Yann LeCun and Amir Bar and Saining Xie},
+  year={2025},
+  eprint={2504.01017},
+  archivePrefix={arXiv},
+  primaryClass={cs.CV}
+}
+```

config.json

@@ -0,0 +1,77 @@
+{
+  "apply_layernorm": true,
+  "architectures": [
+    "Dinov2Model"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "block_chunks": 4,
+  "drop_path_rate": 0.0,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 1536,
+  "image_size": 224,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "layerscale_value": 1.0,
+  "mlp_ratio": 4,
+  "model_type": "dinov2",
+  "num_attention_heads": 24,
+  "num_channels": 3,
+  "num_hidden_layers": 40,
+  "num_layers_per_block": 10,
+  "out_features": [
+    "stage40"
+  ],
+  "out_indices": [
+    40
+  ],
+  "patch_size": 14,
+  "qkv_bias": true,
+  "reshape_hidden_states": true,
+  "stage_names": [
+    "stem",
+    "stage1",
+    "stage2",
+    "stage3",
+    "stage4",
+    "stage5",
+    "stage6",
+    "stage7",
+    "stage8",
+    "stage9",
+    "stage10",
+    "stage11",
+    "stage12",
+    "stage13",
+    "stage14",
+    "stage15",
+    "stage16",
+    "stage17",
+    "stage18",
+    "stage19",
+    "stage20",
+    "stage21",
+    "stage22",
+    "stage23",
+    "stage24",
+    "stage25",
+    "stage26",
+    "stage27",
+    "stage28",
+    "stage29",
+    "stage30",
+    "stage31",
+    "stage32",
+    "stage33",
+    "stage34",
+    "stage35",
+    "stage36",
+    "stage37",
+    "stage38",
+    "stage39",
+    "stage40"
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.2",
+  "use_swiglu_ffn": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df2b9b861736e630d0eb223c6d8809a94fa6ca54206c05634b56f3dbd9bf607d
+size 4539167152

preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "BitImageProcessor",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}