First two transformers for the Polish trained in Clarin-Biz project (#1)

* add docker

* docker update

* ignore

* test transformers loading

* use our v1 polish roberta from hugging face repo

* update clarin hf repo

* update clarin long former model

* add transformers

* move to experimental

* fix moved init

* fix moved init

* add transformers

move to experimental

fix moved init

fix moved init

* mains

* ignore lib without type in mypy
add tensorflow to poetry

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+.idea/

docker/embeddings-gpu/Dockerfile

@@ -0,0 +1,19 @@
+FROM nvidia/cuda:11.0.3-cudnn8-runtime-ubuntu18.04
+MAINTAINER Lukasz Augustyniak <lukasz.augustyniak@pwr.edu.pl>
+
+RUN apt update && \
+    apt install -y bash \
+                   build-essential \
+                   git \
+                   curl \
+                   ca-certificates \
+                   python3 \
+                   python3-pip && \
+    rm -rf /var/lib/apt/lists
+
+RUN python3 -m pip install --no-cache-dir --upgrade pip && \
+    python3 -m pip install --no-cache-dir \
+    jupyter \
+    tensorflow \
+    torch \
+    transformers

experimental/embeddings/language_models/long_former.py

@@ -0,0 +1,24 @@
+from transformers import AutoTokenizer, AutoModel
+import tensorflow as tf
+
+
+def main() -> None:
+    tokenizer = AutoTokenizer.from_pretrained("clarin-pl/long-former-polish")
+    model = AutoModel.from_pretrained("clarin-pl/long-former-polish")
+
+    prompt = "Zażółć gęślą jaźń"
+    choice0 = "Gęś"
+    choice1 = "Jaźń."
+
+    encoding = tokenizer(
+        [[prompt, prompt], [choice0, choice1]], return_tensors="tf", padding=True
+    )
+    inputs = {k: tf.expand_dims(v, 0) for k, v in encoding.items()}
+    outputs = model(inputs)  # batch size is 1
+
+    # the linear classifier still needs to be trained
+    logits = outputs.logits
+
+
+if __name__ == "__main__":
+    main()

experimental/embeddings/language_models/roberta.py

@@ -0,0 +1,13 @@
+from transformers import AutoTokenizer, AutoModel
+
+
+def main() -> None:
+    tokenizer = AutoTokenizer.from_pretrained("clarin-pl/roberta-polish-kgr10")
+    model = AutoModel.from_pretrained("clarin-pl/roberta-polish-kgr10")
+
+    encoded_input = tokenizer.encode("Zażółć gęślą jaźń", return_tensors="pt")
+    outputs = model(encoded_input)
+
+
+if __name__ == "__main__":
+    main()

mypy.ini

@@ -0,0 +1,5 @@
+[mypy-transformers.*]
+ignore_missing_imports = True
+
+[mypy-tensorflow.*]
+ignore_missing_imports = True