Source code for tango.integrations.transformers.tokenizer
from transformers import AutoTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase
from tango.common import Registrable
[docs]class Tokenizer(PreTrainedTokenizerBase, Registrable):
"""
A :class:`~tango.common.Registrable` version of transformers'
:class:`~transformers.PreTrainedTokenizerBase`.
"""
default_implementation = "auto"
"""
The default registered implementation just calls
:meth:`transformers.AutoTokenizer.from_pretrained()`.
"""
Tokenizer.register("auto", constructor="from_pretrained")(AutoTokenizer)