Publish Pixel Translation checkpoints

.gitattributes

@@ -0,0 +1 @@
+*.bin filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,65 @@
+---
+library_name: pytorch
+pipeline_tag: translation
+tags:
+- arabic
+- english
+- machine-translation
+---
+
+# Pixel Translation checkpoints
+
+Final checkpoints for the Base, Vocab+, Byte, and Contextual Pixel Arabic-to-English
+translation models. These weights are intended for use with
+[mbzuai-nlp/arapixel-mt](https://github.com/mbzuai-nlp/arapixel-mt).
+
+## Layout
+
+Each model has MSA-pretrained checkpoints for 5%, 10%, 25%, 50%, and the full
+MSA training set:
+
+```text
+<model>/msa/{5pct,10pct,25pct,50pct,full}/pytorch_model.bin
+```
+
+The curriculum checkpoints were fine-tuned on the 1:1 MSA/dialect mixture:
+
+```text
+<model>/finetuned/{5pct,10pct,25pct,50pct}/msa_dialect_1x/best_model.bin
+```
+
+Full-data checkpoints have four fine-tuning variants:
+
+```text
+<model>/finetuned/full/dialect/best_model.bin
+<model>/finetuned/full/msa_dialect_1x/best_model.bin
+<model>/finetuned/full/msa_dialect_2x/best_model.bin
+<model>/finetuned/full/msa_dialect_4x/best_model.bin
+```
+
+`<model>` is one of `pixel`, `bytes`, `vocab`, or `base`. In the codebase,
+the Base model is selected with `--model decoder`.
+
+Every checkpoint directory also contains `training_complete.json`, which lets
+evaluation select the fixed MSA or dynamic fine-tuning source layout. The shared
+Vocab+ tokenizer is stored in `vocab/tokenizer/`.
+
+## Download one checkpoint
+
+```bash
+hf download shaheen6/PixelTranslationCheckpoints   --include "pixel/msa/full/*"   --local-dir checkpoints
+```
+
+The downloaded model path is then:
+
+```text
+checkpoints/pixel/msa/full/pytorch_model.bin
+```
+
+Pass that file to `--model-checkpoint`. For Vocab+, also download the tokenizer:
+
+```bash
+hf download shaheen6/PixelTranslationCheckpoints   --include "vocab/tokenizer/*"   --local-dir checkpoints
+
+export VOCAB_TOKENIZER_DIR="$PWD/checkpoints/vocab/tokenizer"
+```

base/finetuned/10pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cda033504cded8b3404d803e205d4fd1a5e27e200d3969f439e58abbb0a7941a
+size 1447436146

base/finetuned/10pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "10pct",
+  "weight_bytes": 1447436146,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

base/finetuned/25pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7784b9b9a096bd73b8cece4bd02366b2884f029050e6222456fbb287d3a601f
+size 1447436146

base/finetuned/25pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "25pct",
+  "weight_bytes": 1447436146,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

base/finetuned/50pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aeea707d355d1f1f46f42b13ef423762332ee0ceaee7f8a44159a5cd7abf0ed
+size 1447436146

base/finetuned/50pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "50pct",
+  "weight_bytes": 1447436146,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

base/finetuned/5pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e13636e366b4a91d98c3b2b2ee63bb784feb921c99260df8956739d718d74ad7
+size 1447436146

base/finetuned/5pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "5pct",
+  "weight_bytes": 1447436146,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

base/finetuned/full/dialect/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c7c0e782527bdd92e187d185ec10cfde105d1fda8a01b951c235ccbbb026a8b
+size 1447433158

base/finetuned/full/dialect/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "full",
+  "weight_bytes": 1447433158,
+  "training_phase": "finetuning",
+  "mixture": "Dialect_Combined",
+  "source_layout": "dynamic"
+}

base/finetuned/full/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85fc095f44cbfe793191eca0d0bf1075417c0fcd64cdabc1b241ad20b4025f2f
+size 1447433158

base/finetuned/full/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "full",
+  "weight_bytes": 1447433158,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

base/finetuned/full/msa_dialect_2x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b74e73e5df560adf275386888c6940b65fc09f6dc60efef2a8cf8cf065a6b292
+size 1447433158

base/finetuned/full/msa_dialect_2x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "full",
+  "weight_bytes": 1447433158,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_2x",
+  "source_layout": "dynamic"
+}

base/finetuned/full/msa_dialect_4x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59d5f112389f36ea57c52c9ca6dd72e8f6c45bc7ae797f688969781e08f47ec7
+size 1447433158

base/finetuned/full/msa_dialect_4x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "full",
+  "weight_bytes": 1447433158,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_4x",
+  "source_layout": "dynamic"
+}

base/msa/10pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddcee1cc8e38c87a9d41d1c1d9d8ed790ac66cda9dc3e97ed84a1e1ae9cd64a5
+size 1447437206

base/msa/10pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "10pct",
+  "weight_bytes": 1447437206,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

base/msa/25pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14b4fdb8b6d9130e9cd2b460052d483d40ff8eb274b0ad92130a644350530af4
+size 1447437206

base/msa/25pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "25pct",
+  "weight_bytes": 1447437206,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

base/msa/50pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef77f63b874315f0058e88c99eef81d5f3a7ead7975805c83424a8e1cebf113d
+size 1447437206

base/msa/50pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "50pct",
+  "weight_bytes": 1447437206,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

base/msa/5pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b73f80b6e27980cbbc97a7a406aebc1586bc6edd8fe97df9cc23043ffefae20
+size 1447437206

base/msa/5pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "5pct",
+  "weight_bytes": 1447437206,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

base/msa/full/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e69918cd7508dbed28360d13c5694bccbcc76f6fe79fad6f639422ffa163e519
+size 1447434154

base/msa/full/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "decoder",
+  "release_model": "base",
+  "pretraining_data": "full",
+  "weight_bytes": 1447434154,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

bytes/finetuned/10pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0501f2605ed7103e42746925ac8c3c061468a353c5f8b9fbddb0be8c25fe35c
+size 1933292724

bytes/finetuned/10pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "10pct",
+  "weight_bytes": 1933292724,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

bytes/finetuned/25pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb322bfbbd395e5282bb73905c10d00fffa0ac7766ade127fd8a646461a80b72
+size 1933292724

bytes/finetuned/25pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "25pct",
+  "weight_bytes": 1933292724,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

bytes/finetuned/50pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:138cfd5b2e2b0515969e6254e59715a7f6726a30af57d9d5a1cf2a7a9e574d24
+size 1933292724

bytes/finetuned/50pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "50pct",
+  "weight_bytes": 1933292724,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

bytes/finetuned/5pct/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b3eea142d441f9da322ae3d13e11f4ade3cf56d48675b2a4d6dbb9314a1adea
+size 1933292724

bytes/finetuned/5pct/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "5pct",
+  "weight_bytes": 1933292724,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

bytes/finetuned/full/dialect/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae40063393fbc8844deaa6e6fc7cb182750d0637a62ced9536fd62b761b9069b
+size 1933289043

bytes/finetuned/full/dialect/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "full",
+  "weight_bytes": 1933289043,
+  "training_phase": "finetuning",
+  "mixture": "Dialect_Combined",
+  "source_layout": "dynamic"
+}

bytes/finetuned/full/msa_dialect_1x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2362f12c7078adcd4f186934cf4423cdfe50bedcb76de4fbad0bed19e530262b
+size 1933289043

bytes/finetuned/full/msa_dialect_1x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "full",
+  "weight_bytes": 1933289043,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_1x",
+  "source_layout": "dynamic"
+}

bytes/finetuned/full/msa_dialect_2x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc255e487d92d452a5e784282b57c50c53aecf85a04a81e4486b9c670389f682
+size 1933289043

bytes/finetuned/full/msa_dialect_2x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "full",
+  "weight_bytes": 1933289043,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_2x",
+  "source_layout": "dynamic"
+}

bytes/finetuned/full/msa_dialect_4x/best_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8230592c65fa0ed83693d5baa4abeab9f86eab47d4289d4436992c37b7debc9
+size 1933289043

bytes/finetuned/full/msa_dialect_4x/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "full",
+  "weight_bytes": 1933289043,
+  "training_phase": "finetuning",
+  "mixture": "MSA_Dialect_Combined_4x",
+  "source_layout": "dynamic"
+}

bytes/msa/10pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f86f2ed0389cf8915d138e3bdbac9508abbe2abb5ac524749a5980249dc39f
+size 1933294015

bytes/msa/10pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "10pct",
+  "weight_bytes": 1933294015,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

bytes/msa/25pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9c16576f4c29df9d8a7f27dd1f0499b8cc8c2fb7fae227bc3d29258169c6d77
+size 1933294015

bytes/msa/25pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "25pct",
+  "weight_bytes": 1933294015,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}

bytes/msa/50pct/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d577580b6f1fd1e18d619addda1e9bb4e5a9d44e13bc21eba32526baa48caa8
+size 1933294015

bytes/msa/50pct/training_complete.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "bytes",
+  "release_model": "bytes",
+  "pretraining_data": "50pct",
+  "weight_bytes": 1933294015,
+  "stage": "full",
+  "published_stage": "full_epoch2",
+  "source_layout": "msa"
+}