builddataset.py

import numpy as np
from collections import Counter

def load_data(data_path):
    data = []
    sentence = []
    labels = []
    num_words = 0 
    for line in open(data_path, 'r'):
        line = line.strip()
        if line == '':
            if len(sentence) > 0:
                data.append((sentence, labels))
                sentence = []
                labels = []
        else:
            word, label = line.split()
            num_words += 1
            sentence.append(word)
            labels.append(label)
    if len(sentence) > 0:
        data.append((sentence, labels))
    print("total words num:", num_words)
    print("total sentences num:", len(data))
    return data

def build_label_vocab(slot_names_path):
    labels = ['O']
    for line in open(slot_names_path, 'r'):
        line = line.strip()
        labels.append("B-"+line)
        labels.append("I-"+line)

    label2idx = dict(zip(labels, range(len(labels))))
    idx2label = dict(zip(range(len(labels)), labels))
    return label2idx, idx2label

def build_vocab(data, min_count=1):
    count = [("<UNK>", -1), ("<PAD>", -1)]
    words = []
    for sentence, _ in data: 
        words.extend(sentence)
  
    counter = Counter(words)
    counter_list = counter.most_common()
    for word, c in counter_list:
        if c >= min_count:
            count.append((word, c))
    word2idx = dict()
    for word, _ in count:
        word2idx[word] = len(word2idx)
    idx2word = dict(zip(word2idx.values(), word2idx.keys()))
  
    return word2idx, idx2word

def build_dataset(data, word2idx, label2idx):
    num_text = []
    num_label = []
    for sentence, label in data:
        num_text.append(np.array([word2idx[w] for w in sentence], dtype=np.int64))
        num_label.append(np.array([label2idx[l] for l in label], dtype=np.int64))
    return num_text, num_label