You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
102 lines
3.5 KiB
102 lines
3.5 KiB
# !/usr/bin/env python3 |
|
# -*- coding: utf-8 -*- |
|
# |
|
# Copyright 2016-2099 Ailemon.net |
|
# |
|
# This file is part of ASRT Speech Recognition Tool. |
|
# |
|
# ASRT is free software: you can redistribute it and/or modify |
|
# it under the terms of the GNU General Public License as published by |
|
# the Free Software Foundation, either version 3 of the License, or |
|
# (at your option) any later version. |
|
# ASRT is distributed in the hope that it will be useful, |
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
# GNU General Public License for more details. |
|
# |
|
# You should have received a copy of the GNU General Public License |
|
# along with ASRT. If not, see <https://www.gnu.org/licenses/>. |
|
# ============================================================================ |
|
|
|
""" |
|
@author: nl8590687 |
|
加载语音数据集用的数据加载器的定义 |
|
""" |
|
import os |
|
import random |
|
import numpy as np |
|
from utils.config import load_config_file, DEFAULT_CONFIG_FILENAME, load_pinyin_dict |
|
from utils.ops import read_wav_data |
|
|
|
|
|
class DataLoader: |
|
""" |
|
数据加载器 |
|
|
|
参数：\\ |
|
config: 配置信息字典 |
|
dataset_type: 要加载的数据集类型，包含('train', 'dev', 'test')三种 |
|
""" |
|
def __init__(self, dataset_type:str): |
|
self.dataset_type = dataset_type |
|
|
|
self.data_list = list() |
|
self.wav_dict = dict() |
|
self.label_dict = dict() |
|
self.pinyin_list = list() |
|
self.pinyin_dict = dict() |
|
self._load_data() |
|
|
|
def _load_data(self): |
|
config = load_config_file(DEFAULT_CONFIG_FILENAME) |
|
|
|
self.pinyin_list, self.pinyin_dict = load_pinyin_dict(config['dict_filename']) |
|
|
|
for index in range(len(config['dataset'][self.dataset_type])): |
|
filename_datalist = config['dataset'][self.dataset_type][index]['data_list'] |
|
filename_datapath = config['dataset'][self.dataset_type][index]['data_path'] |
|
with open(filename_datalist, 'r', encoding='utf-8') as file_pointer: |
|
lines = file_pointer.read().split('\n') |
|
for line in lines: |
|
if len(line) == 0: |
|
continue |
|
tokens = line.split(' ') |
|
self.data_list.append(tokens[0]) |
|
self.wav_dict[tokens[0]] = os.path.join(filename_datapath, tokens[1]) |
|
|
|
filename_labellist = config['dataset'][self.dataset_type][index]['label_list'] |
|
with open(filename_labellist, 'r', encoding='utf-8') as file_pointer: |
|
lines = file_pointer.read().split('\n') |
|
for line in lines: |
|
if len(line) == 0: |
|
continue |
|
tokens = line.split(' ') |
|
self.label_dict[tokens[0]] = tokens[1:] |
|
|
|
def get_data_count(self) -> int: |
|
""" |
|
获取数据集总数量 |
|
""" |
|
return len(self.data_list) |
|
|
|
def get_data(self, index: int) -> tuple: |
|
""" |
|
按下标获取一条数据 |
|
""" |
|
mark = self.data_list[index] |
|
|
|
wav_signal, sample_rate, _, _ = read_wav_data(self.wav_dict[mark]) |
|
labels = list() |
|
for item in self.label_dict[mark]: |
|
if len(item) == 0: |
|
continue |
|
labels.append(self.pinyin_dict[item]) |
|
|
|
data_label = np.array(labels) |
|
return wav_signal, sample_rate, data_label |
|
|
|
def shuffle(self) -> None: |
|
""" |
|
随机打乱数据 |
|
""" |
|
random.shuffle(self.data_list)
|
|
|