vocab_generator

Author: Heli Qi Affiliation: NAIST Date: 2022.07

generate_vocab_word(save_path, text_path, txt_format, vocab_size)

This function just segments text with whitespaces, so the punctuation symbols won't be treated as independent tokens.

Source code in speechain/datasets/pyscripts/vocab_generator.py

def generate_vocab_word(
    save_path: str, text_path: str, txt_format: str, vocab_size: int
):
    """
    This function just segments text with whitespaces, so the punctuation symbols won't be treated as independent tokens.

    """
    # --- Vocabulary List Generation --- #
    save_token_vocab(
        save_path=save_path,
        text_path=text_path,
        txt_format=txt_format,
        text2tokens_func=text2word_list,
        vocab_size=vocab_size,
        save_idx2text_len=True,
    )

save_token_vocab(save_path, text_path, txt_format, text2tokens_func, vocab_size, save_idx2text=False, save_idx2text_len=False)

Obtain and save the token vocabulary for char and word tokenizers. The tokens in the vocabulary are sorted in descending order by their occurrence frequency in the text data.

Parameters:

Name	Type	Description	Default
save_path	str	str Where to save the token vocabulary	required
text_path	str	str Where the text data used to get the vocabulary is placed	required
txt_format	str	str	required
text2tokens_func	callable	function The function that transforms a sentence string into a list of tokens	required
vocab_size	int or str	int The maximum number of tokens in the vocabulary. Useless if vocab_size is larger than the actual token number.	required
save_idx2text	bool	bool = False	False
save_idx2text_len	bool	bool = False	False

Returns:

Type	Description
	The modified save_path where the tokenizer configuration is attached at the end

Source code in speechain/datasets/pyscripts/vocab_generator.py

def save_token_vocab(
    save_path: str,
    text_path: str,
    txt_format: str,
    text2tokens_func: callable,
    vocab_size: int or str,
    save_idx2text: bool = False,
    save_idx2text_len: bool = False,
):
    """
    Obtain and save the token vocabulary for char and word tokenizers.
    The tokens in the vocabulary are sorted in descending order by their occurrence frequency in the text data.

    Args:
        save_path: str
            Where to save the token vocabulary
        text_path: str
            Where the text data used to get the vocabulary is placed
        txt_format: str

        text2tokens_func: function
            The function that transforms a sentence string into a list of tokens
        vocab_size: int
            The maximum number of tokens in the vocabulary.
            Useless if vocab_size is larger than the actual token number.
        save_idx2text: bool = False
        save_idx2text_len: bool = False

    Returns:
        The modified save_path where the tokenizer configuration is attached at the end

    """
    # --- 1. Data Initialization --- #
    if isinstance(vocab_size, int):
        save_path = os.path.join(save_path, get_readable_number(vocab_size), txt_format)
    else:
        save_path = os.path.join(
            save_path, "full_tokens" if vocab_size is None else vocab_size, txt_format
        )
    os.makedirs(save_path, exist_ok=True)

    vocab_path = os.path.join(save_path, "vocab")
    if not os.path.exists(vocab_path):
        # read the index-to-text file and turn the information into a Dict
        idx2text = load_idx2data_file(os.path.join(text_path, f"idx2{txt_format}_text"))
        # convert each string sentence into its token sentence
        idx2text_token = {
            idx: text2tokens_func(text) for idx, text in tqdm(idx2text.items())
        }

        # --- 2. Token Vocabulary Saving --- #
        # gather the tokens of all the sentences into a single list
        tokens = []
        for value in idx2text_token.values():
            tokens += value
        # collect the occurrence frequency of each token
        token2freq = sorted(Counter(tokens).items(), key=lambda x: x[1], reverse=True)
        token_vocab = [token for token, _ in token2freq]
        # change the token list and saving path only when the number of token_vocab is larger than vocab_size
        if isinstance(vocab_size, int) and len(token_vocab) >= vocab_size - 3:
            token_vocab = token_vocab[: vocab_size - 3]

        # 0 is designed for the blank (the padding index)
        # -2 is designed for the unknowns
        # -1 is designed for the beginning and end of sentence
        token_vocab = ["<blank>"] + token_vocab + ["<unk>", "<sos/eos>"]
        np.savetxt(vocab_path, token_vocab, fmt="%s")
        print(f"Token vocabulary has been successfully saved to {vocab_path}.")

        # --- 3. Tokenized Text and its Length Saving --- #
        if save_idx2text:
            text_token_path = os.path.join(save_path, "idx2text")
            np.savetxt(
                text_token_path,
                [[idx, str(text_token)] for idx, text_token in idx2text_token.items()],
                fmt="%s",
            )
            print(f"Tokenized text has been successfully saved to {text_token_path}.")

        if save_idx2text_len:
            text_len_path = os.path.join(save_path, "idx2text_len")
            np.savetxt(
                text_len_path,
                [[idx, len(text_token)] for idx, text_token in idx2text_token.items()],
                fmt="%s",
            )
            print(
                f"The length of tokenized text has been successfully saved to {text_len_path}."
            )