mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 23:33:05 +04:00
Merge branch 4.x
This commit is contained in:
@@ -0,0 +1,587 @@
|
||||
#include <opencv2/core.hpp>
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/dnn.hpp>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <cmath>
|
||||
#include <random>
|
||||
#include <numeric>
|
||||
using namespace cv;
|
||||
using namespace std;
|
||||
|
||||
class FilterbankFeatures {
|
||||
|
||||
// Initializes pre-processing class. Default values are the values used by the Jasper
|
||||
// architecture for pre-processing. For more details, refer to the paper here:
|
||||
// https://arxiv.org/abs/1904.03288
|
||||
|
||||
private:
|
||||
int sample_rate = 16000;
|
||||
double window_size = 0.02;
|
||||
double window_stride = 0.01;
|
||||
int win_length = static_cast<int>(sample_rate * window_size); // Number of samples in window
|
||||
int hop_length = static_cast<int>(sample_rate * window_stride); // Number of steps to advance between frames
|
||||
int n_fft = 512; // Size of window for STFT
|
||||
|
||||
// Parameters for filterbanks calculation
|
||||
int n_filt = 64;
|
||||
double lowfreq = 0.;
|
||||
double highfreq = sample_rate / 2;
|
||||
|
||||
public:
|
||||
// Mel filterbanks preparation
|
||||
double hz_to_mel(double frequencies)
|
||||
{
|
||||
//Converts frequencies from hz to mel scale
|
||||
// Fill in the linear scale
|
||||
double f_min = 0.0;
|
||||
double f_sp = 200.0 / 3;
|
||||
double mels = (frequencies - f_min) / f_sp;
|
||||
// Fill in the log-scale part
|
||||
double min_log_hz = 1000.0; // beginning of log region (Hz)
|
||||
double min_log_mel = (min_log_hz - f_min) / f_sp; // same (Mels)
|
||||
double logstep = std::log(6.4) / 27.0; // step size for log region
|
||||
|
||||
if (frequencies >= min_log_hz)
|
||||
{
|
||||
mels = min_log_mel + std::log(frequencies / min_log_hz) / logstep;
|
||||
}
|
||||
return mels;
|
||||
}
|
||||
|
||||
vector<double> mel_to_hz(vector<double>& mels)
|
||||
{
|
||||
// Converts frequencies from mel to hz scale
|
||||
|
||||
// Fill in the linear scale
|
||||
double f_min = 0.0;
|
||||
double f_sp = 200.0 / 3;
|
||||
vector<double> freqs;
|
||||
for (size_t i = 0; i < mels.size(); i++)
|
||||
{
|
||||
freqs.push_back(f_min + f_sp * mels[i]);
|
||||
}
|
||||
|
||||
// And now the nonlinear scale
|
||||
double min_log_hz = 1000.0; // beginning of log region (Hz)
|
||||
double min_log_mel = (min_log_hz - f_min) / f_sp; // same (Mels)
|
||||
double logstep = std::log(6.4) / 27.0; // step size for log region
|
||||
|
||||
for(size_t i = 0; i < mels.size(); i++)
|
||||
{
|
||||
if (mels[i] >= min_log_mel)
|
||||
{
|
||||
freqs[i] = min_log_hz * exp(logstep * (mels[i] - min_log_mel));
|
||||
}
|
||||
}
|
||||
return freqs;
|
||||
}
|
||||
|
||||
vector<double> mel_frequencies(int n_mels, double fmin, double fmax)
|
||||
{
|
||||
// Calculates n mel frequencies between 2 frequencies
|
||||
double min_mel = hz_to_mel(fmin);
|
||||
double max_mel = hz_to_mel(fmax);
|
||||
|
||||
vector<double> mels;
|
||||
double step = (max_mel - min_mel) / (n_mels - 1);
|
||||
for(double i = min_mel; i < max_mel; i += step)
|
||||
{
|
||||
mels.push_back(i);
|
||||
}
|
||||
mels.push_back(max_mel);
|
||||
|
||||
vector<double> res = mel_to_hz(mels);
|
||||
return res;
|
||||
}
|
||||
|
||||
vector<vector<double>> mel(int n_mels, double fmin, double fmax)
|
||||
{
|
||||
// Generates mel filterbank matrix
|
||||
|
||||
double num = 1 + n_fft / 2;
|
||||
vector<vector<double>> weights(n_mels, vector<double>(static_cast<int>(num), 0.));
|
||||
|
||||
// Center freqs of each FFT bin
|
||||
vector<double> fftfreqs;
|
||||
double step = (sample_rate / 2) / (num - 1);
|
||||
for(double i = 0; i <= sample_rate / 2; i += step)
|
||||
{
|
||||
fftfreqs.push_back(i);
|
||||
}
|
||||
// 'Center freqs' of mel bands - uniformly spaced between limits
|
||||
vector<double> mel_f = mel_frequencies(n_mels + 2, fmin, fmax);
|
||||
|
||||
vector<double> fdiff;
|
||||
for(size_t i = 1; i < mel_f.size(); ++i)
|
||||
{
|
||||
fdiff.push_back(mel_f[i]- mel_f[i - 1]);
|
||||
}
|
||||
|
||||
vector<vector<double>> ramps(mel_f.size(), vector<double>(fftfreqs.size()));
|
||||
for (size_t i = 0; i < mel_f.size(); ++i)
|
||||
{
|
||||
for (size_t j = 0; j < fftfreqs.size(); ++j)
|
||||
{
|
||||
ramps[i][j] = mel_f[i] - fftfreqs[j];
|
||||
}
|
||||
}
|
||||
|
||||
double lower, upper, enorm;
|
||||
for (int i = 0; i < n_mels; ++i)
|
||||
{
|
||||
// using Slaney-style mel which is scaled to be approx constant energy per channel
|
||||
enorm = 2./(mel_f[i + 2] - mel_f[i]);
|
||||
|
||||
for (int j = 0; j < static_cast<int>(num); ++j)
|
||||
{
|
||||
// lower and upper slopes for all bins
|
||||
lower = (-1) * ramps[i][j] / fdiff[i];
|
||||
upper = ramps[i + 2][j] / fdiff[i + 1];
|
||||
|
||||
weights[i][j] = max(0., min(lower, upper)) * enorm;
|
||||
}
|
||||
}
|
||||
return weights;
|
||||
}
|
||||
|
||||
// STFT preparation
|
||||
vector<double> pad_window_center(vector<double>&data, int size)
|
||||
{
|
||||
// Pad the window out to n_fft size
|
||||
int n = static_cast<int>(data.size());
|
||||
int lpad = static_cast<int>((size - n) / 2);
|
||||
vector<double> pad_array;
|
||||
|
||||
for(int i = 0; i < lpad; ++i)
|
||||
{
|
||||
pad_array.push_back(0.);
|
||||
}
|
||||
|
||||
for(size_t i = 0; i < data.size(); ++i)
|
||||
{
|
||||
pad_array.push_back(data[i]);
|
||||
}
|
||||
|
||||
for(int i = 0; i < lpad; ++i)
|
||||
{
|
||||
pad_array.push_back(0.);
|
||||
}
|
||||
return pad_array;
|
||||
}
|
||||
|
||||
vector<vector<double>> frame(vector<double>& x)
|
||||
{
|
||||
// Slices a data array into overlapping frames.
|
||||
int n_frames = static_cast<int>(1 + (x.size() - n_fft) / hop_length);
|
||||
vector<vector<double>> new_x(n_fft, vector<double>(n_frames));
|
||||
|
||||
for (int i = 0; i < n_fft; ++i)
|
||||
{
|
||||
for (int j = 0; j < n_frames; ++j)
|
||||
{
|
||||
new_x[i][j] = x[i + j * hop_length];
|
||||
}
|
||||
}
|
||||
return new_x;
|
||||
}
|
||||
|
||||
vector<double> hanning()
|
||||
{
|
||||
// https://en.wikipedia.org/wiki/Window_function#Hann_and_Hamming_windows
|
||||
vector<double> window_tensor;
|
||||
for (int j = 1 - win_length; j < win_length; j+=2)
|
||||
{
|
||||
window_tensor.push_back(1 - (0.5 * (1 - cos(CV_PI * j / (win_length - 1)))));
|
||||
}
|
||||
return window_tensor;
|
||||
}
|
||||
|
||||
vector<vector<double>> stft_power(vector<double>& y)
|
||||
{
|
||||
// Short Time Fourier Transform. The STFT represents a signal in the time-frequency
|
||||
// domain by computing discrete Fourier transforms (DFT) over short overlapping windows.
|
||||
// https://en.wikipedia.org/wiki/Short-time_Fourier_transform
|
||||
|
||||
// Pad the time series so that frames are centered
|
||||
vector<double> new_y;
|
||||
int num = int(n_fft / 2);
|
||||
|
||||
for (int i = 0; i < num; ++i)
|
||||
{
|
||||
new_y.push_back(y[num - i]);
|
||||
}
|
||||
for (size_t i = 0; i < y.size(); ++i)
|
||||
{
|
||||
new_y.push_back(y[i]);
|
||||
}
|
||||
for (size_t i = y.size() - 2; i >= y.size() - num - 1; --i)
|
||||
{
|
||||
new_y.push_back(y[i]);
|
||||
}
|
||||
|
||||
// Compute a window function
|
||||
vector<double> window_tensor = hanning();
|
||||
|
||||
// Pad the window out to n_fft size
|
||||
vector<double> fft_window = pad_window_center(window_tensor, n_fft);
|
||||
|
||||
// Window the time series
|
||||
vector<vector<double>> y_frames = frame(new_y);
|
||||
|
||||
// Multiply on fft_window
|
||||
for (size_t i = 0; i < y_frames.size(); ++i)
|
||||
{
|
||||
for (size_t j = 0; j < y_frames[0].size(); ++j)
|
||||
{
|
||||
y_frames[i][j] *= fft_window[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Transpose frames for computing stft
|
||||
vector<vector<double>> y_frames_transpose(y_frames[0].size(), vector<double>(y_frames.size()));
|
||||
for (size_t i = 0; i < y_frames[0].size(); ++i)
|
||||
{
|
||||
for (size_t j = 0; j < y_frames.size(); ++j)
|
||||
{
|
||||
y_frames_transpose[i][j] = y_frames[j][i];
|
||||
}
|
||||
}
|
||||
|
||||
// Short Time Fourier Transform
|
||||
// and get power of spectrum
|
||||
vector<vector<double>> spectrum_power(y_frames_transpose[0].size() / 2 + 1 );
|
||||
for (size_t i = 0; i < y_frames_transpose.size(); ++i)
|
||||
{
|
||||
Mat dstMat;
|
||||
dft(y_frames_transpose[i], dstMat, DFT_COMPLEX_OUTPUT);
|
||||
|
||||
// we need only the first part of the spectrum, the second part is symmetrical
|
||||
for (int j = 0; j < static_cast<int>(y_frames_transpose[0].size()) / 2 + 1; ++j)
|
||||
{
|
||||
double power_re = dstMat.at<double>(2 * j) * dstMat.at<double>(2 * j);
|
||||
double power_im = dstMat.at<double>(2 * j + 1) * dstMat.at<double>(2 * j + 1);
|
||||
spectrum_power[j].push_back(power_re + power_im);
|
||||
}
|
||||
}
|
||||
return spectrum_power;
|
||||
}
|
||||
|
||||
Mat calculate_features(vector<double>& x)
|
||||
{
|
||||
// Calculates filterbank features matrix.
|
||||
|
||||
// Do preemphasis
|
||||
std::default_random_engine generator;
|
||||
std::normal_distribution<double> normal_distr(0, 1);
|
||||
double dither = 1e-5;
|
||||
for(size_t i = 0; i < x.size(); ++i)
|
||||
{
|
||||
x[i] += dither * static_cast<double>(normal_distr(generator));
|
||||
}
|
||||
double preemph = 0.97;
|
||||
for (size_t i = x.size() - 1; i > 0; --i)
|
||||
{
|
||||
x[i] -= preemph * x[i-1];
|
||||
}
|
||||
|
||||
// Calculate Short Time Fourier Transform and get power of spectrum
|
||||
auto spectrum_power = stft_power(x);
|
||||
|
||||
vector<vector<double>> filterbanks = mel(n_filt, lowfreq, highfreq);
|
||||
|
||||
// Calculate log of multiplication of filterbanks matrix on spectrum_power matrix
|
||||
vector<vector<double>> x_stft(filterbanks.size(), vector<double>(spectrum_power[0].size(), 0));
|
||||
|
||||
for (size_t i = 0; i < filterbanks.size(); ++i)
|
||||
{
|
||||
for (size_t j = 0; j < filterbanks[0].size(); ++j)
|
||||
{
|
||||
for (size_t k = 0; k < spectrum_power[0].size(); ++k)
|
||||
{
|
||||
x_stft[i][k] += filterbanks[i][j] * spectrum_power[j][k];
|
||||
}
|
||||
}
|
||||
for (size_t k = 0; k < spectrum_power[0].size(); ++k)
|
||||
{
|
||||
x_stft[i][k] = std::log(x_stft[i][k] + 1e-20);
|
||||
}
|
||||
}
|
||||
|
||||
// normalize data
|
||||
auto elments_num = x_stft[0].size();
|
||||
for(size_t i = 0; i < x_stft.size(); ++i)
|
||||
{
|
||||
double x_mean = std::accumulate(x_stft[i].begin(), x_stft[i].end(), 0.) / elments_num; // arithmetic mean
|
||||
double x_std = 0; // standard deviation
|
||||
for(size_t j = 0; j < elments_num; ++j)
|
||||
{
|
||||
double subtract = x_stft[i][j] - x_mean;
|
||||
x_std += subtract * subtract;
|
||||
}
|
||||
x_std /= elments_num;
|
||||
x_std = sqrt(x_std) + 1e-10; // make sure x_std is not zero
|
||||
|
||||
for(size_t j = 0; j < elments_num; ++j)
|
||||
{
|
||||
x_stft[i][j] = (x_stft[i][j] - x_mean) / x_std; // standard score
|
||||
}
|
||||
}
|
||||
|
||||
Mat calculate_features(static_cast<int>(x_stft.size()), static_cast<int>(x_stft[0].size()), CV_32F);
|
||||
for(int i = 0; i < calculate_features.size[0]; ++i)
|
||||
{
|
||||
for(int j = 0; j < calculate_features.size[1]; ++j)
|
||||
{
|
||||
calculate_features.at<float>(i, j) = static_cast<float>(x_stft[i][j]);
|
||||
}
|
||||
}
|
||||
return calculate_features;
|
||||
}
|
||||
};
|
||||
|
||||
class Decoder {
|
||||
// Used for decoding the output of jasper model
|
||||
private:
|
||||
unordered_map<int, char> labels_map = fillMap();
|
||||
int blank_id = 28;
|
||||
|
||||
public:
|
||||
unordered_map<int, char> fillMap()
|
||||
{
|
||||
vector<char> labels={' ','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p'
|
||||
,'q','r','s','t','u','v','w','x','y','z','\''};
|
||||
unordered_map<int, char> map;
|
||||
for(int i = 0; i < static_cast<int>(labels.size()); ++i)
|
||||
{
|
||||
map[i] = labels[i];
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
string decode(Mat& x)
|
||||
{
|
||||
// Takes output of Jasper model and performs ctc decoding algorithm to
|
||||
// remove duplicates and special symbol. Returns prediction
|
||||
|
||||
vector<int> prediction;
|
||||
for(int i = 0; i < x.size[1]; ++i)
|
||||
{
|
||||
double maxEl = -1e10;
|
||||
int ind = 0;
|
||||
for(int j = 0; j < x.size[2]; ++j)
|
||||
{
|
||||
if (maxEl <= x.at<float>(0, i, j))
|
||||
{
|
||||
maxEl = x.at<float>(0, i, j);
|
||||
ind = j;
|
||||
}
|
||||
}
|
||||
prediction.push_back(ind);
|
||||
}
|
||||
// CTC decoding procedure
|
||||
vector<double> decoded_prediction = {};
|
||||
int previous = blank_id;
|
||||
|
||||
for(int i = 0; i < static_cast<int>(prediction.size()); ++i)
|
||||
{
|
||||
if (( prediction[i] != previous || previous == blank_id) && prediction[i] != blank_id)
|
||||
{
|
||||
decoded_prediction.push_back(prediction[i]);
|
||||
}
|
||||
previous = prediction[i];
|
||||
}
|
||||
|
||||
string hypotheses = {};
|
||||
for(size_t i = 0; i < decoded_prediction.size(); ++i)
|
||||
{
|
||||
auto it = labels_map.find(static_cast<char>(decoded_prediction[i]));
|
||||
if (it != labels_map.end())
|
||||
hypotheses.push_back(it->second);
|
||||
}
|
||||
return hypotheses;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
static string predict(Mat& features, dnn::Net net, Decoder decoder)
|
||||
{
|
||||
// Passes the features through the Jasper model and decodes the output to english transcripts.
|
||||
|
||||
// expand 2d features matrix to 3d
|
||||
vector<int> sizes = {1, static_cast<int>(features.size[0]),
|
||||
static_cast<int>(features.size[1])};
|
||||
features = features.reshape(0, sizes);
|
||||
|
||||
// make prediction
|
||||
net.setInput(features);
|
||||
Mat output = net.forward();
|
||||
|
||||
// decode output to transcript
|
||||
auto prediction = decoder.decode(output);
|
||||
return prediction;
|
||||
}
|
||||
|
||||
static int readAudioFile(vector<double>& inputAudio, string file, int audioStream)
|
||||
{
|
||||
VideoCapture cap;
|
||||
int samplingRate = 16000;
|
||||
vector<int> params { CAP_PROP_AUDIO_STREAM, audioStream,
|
||||
CAP_PROP_VIDEO_STREAM, -1,
|
||||
CAP_PROP_AUDIO_DATA_DEPTH, CV_32F,
|
||||
CAP_PROP_AUDIO_SAMPLES_PER_SECOND, samplingRate
|
||||
};
|
||||
cap.open(file, CAP_ANY, params);
|
||||
if (!cap.isOpened())
|
||||
{
|
||||
cerr << "Error : Can't read audio file: '" << file << "' with audioStream = " << audioStream << endl;
|
||||
return -1;
|
||||
}
|
||||
const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX);
|
||||
vector<double> frameVec;
|
||||
Mat frame;
|
||||
for (;;)
|
||||
{
|
||||
if (cap.grab())
|
||||
{
|
||||
cap.retrieve(frame, audioBaseIndex);
|
||||
frameVec = frame;
|
||||
inputAudio.insert(inputAudio.end(), frameVec.begin(), frameVec.end());
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
return samplingRate;
|
||||
}
|
||||
|
||||
static int readAudioMicrophone(vector<double>& inputAudio, int microTime)
|
||||
{
|
||||
VideoCapture cap;
|
||||
int samplingRate = 16000;
|
||||
vector<int> params { CAP_PROP_AUDIO_STREAM, 0,
|
||||
CAP_PROP_VIDEO_STREAM, -1,
|
||||
CAP_PROP_AUDIO_DATA_DEPTH, CV_32F,
|
||||
CAP_PROP_AUDIO_SAMPLES_PER_SECOND, samplingRate
|
||||
};
|
||||
cap.open(0, CAP_ANY, params);
|
||||
if (!cap.isOpened())
|
||||
{
|
||||
cerr << "Error: Can't open microphone" << endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX);
|
||||
vector<double> frameVec;
|
||||
Mat frame;
|
||||
if (microTime <= 0)
|
||||
{
|
||||
cerr << "Error: Duration of audio chunk must be > 0" << endl;
|
||||
return -1;
|
||||
}
|
||||
size_t sizeOfData = static_cast<size_t>(microTime * samplingRate);
|
||||
while (inputAudio.size() < sizeOfData)
|
||||
{
|
||||
if (cap.grab())
|
||||
{
|
||||
cap.retrieve(frame, audioBaseIndex);
|
||||
frameVec = frame;
|
||||
inputAudio.insert(inputAudio.end(), frameVec.begin(), frameVec.end());
|
||||
}
|
||||
else
|
||||
{
|
||||
cerr << "Error: Grab error" << endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return samplingRate;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
const String keys =
|
||||
"{help h usage ? | | This script runs Jasper Speech recognition model }"
|
||||
"{input_file i | | Path to input audio file. If not specified, microphone input will be used }"
|
||||
"{audio_duration t | 15 | Duration of audio chunk to be captured from microphone }"
|
||||
"{audio_stream a | 0 | CAP_PROP_AUDIO_STREAM value }"
|
||||
"{show_spectrogram s | false | Show a spectrogram of the input audio: true / false / 1 / 0 }"
|
||||
"{model m | jasper.onnx | Path to the onnx file of Jasper. You can download the converted onnx model "
|
||||
"from https://drive.google.com/drive/folders/1wLtxyao4ItAg8tt4Sb63zt6qXzhcQoR6?usp=sharing}"
|
||||
"{backend b | dnn::DNN_BACKEND_DEFAULT | Select a computation backend: "
|
||||
"dnn::DNN_BACKEND_DEFAULT, "
|
||||
"dnn::DNN_BACKEND_INFERENCE_ENGINE, "
|
||||
"dnn::DNN_BACKEND_OPENCV }"
|
||||
"{target t | dnn::DNN_TARGET_CPU | Select a target device: "
|
||||
"dnn::DNN_TARGET_CPU, "
|
||||
"dnn::DNN_TARGET_OPENCL, "
|
||||
"dnn::DNN_TARGET_OPENCL_FP16 }"
|
||||
;
|
||||
CommandLineParser parser(argc, argv, keys);
|
||||
if (parser.has("help"))
|
||||
{
|
||||
parser.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Load Network
|
||||
dnn::Net net = dnn::readNetFromONNX(parser.get<std::string>("model"));
|
||||
net.setPreferableBackend(parser.get<int>("backend"));
|
||||
net.setPreferableTarget(parser.get<int>("target"));
|
||||
|
||||
// Get audio
|
||||
vector<double>inputAudio = {};
|
||||
int samplingRate = 0;
|
||||
if (parser.has("input_file"))
|
||||
{
|
||||
string audio = samples::findFile(parser.get<std::string>("input_file"));
|
||||
samplingRate = readAudioFile(inputAudio, audio, parser.get<int>("audio_stream"));
|
||||
}
|
||||
else
|
||||
{
|
||||
samplingRate = readAudioMicrophone(inputAudio, parser.get<int>("audio_duration"));
|
||||
}
|
||||
|
||||
if ((inputAudio.size() == 0) || samplingRate <= 0)
|
||||
{
|
||||
cerr << "Error: problems with audio reading, check input arguments" << endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (inputAudio.size() / samplingRate < 6)
|
||||
{
|
||||
cout << "Warning: For predictable network performance duration of audio must exceed 6 sec."
|
||||
" Audio will be extended with zero samples" << endl;
|
||||
for(int i = static_cast<int>(inputAudio.size()) - 1; i < samplingRate * 6; ++i)
|
||||
{
|
||||
inputAudio.push_back(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate features
|
||||
FilterbankFeatures filter;
|
||||
auto calculated_features = filter.calculate_features(inputAudio);
|
||||
|
||||
// Show spectogram if required
|
||||
if (parser.get<bool>("show_spectrogram") == true)
|
||||
{
|
||||
Mat spectogram;
|
||||
normalize(calculated_features, spectogram, 0, 255, NORM_MINMAX, CV_8U);
|
||||
applyColorMap(spectogram, spectogram, COLORMAP_INFERNO);
|
||||
imshow("spectogram", spectogram);
|
||||
waitKey(0);
|
||||
}
|
||||
|
||||
Decoder decoder;
|
||||
string prediction = predict(calculated_features, net, decoder);
|
||||
for( auto &transcript: prediction)
|
||||
{
|
||||
cout << transcript;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -44,7 +44,7 @@ import os
|
||||
model.graph.initializer.insert(i,init)
|
||||
```
|
||||
|
||||
6. Add an additional reshape node to handle the inconsistant input from python and c++ of openCV.
|
||||
6. Add an additional reshape node to handle the inconsistent input from python and c++ of openCV.
|
||||
see https://github.com/opencv/opencv/issues/19091
|
||||
Make & insert a new node with 'Reshape' operation & required initializer
|
||||
```
|
||||
@@ -256,7 +256,7 @@ class FilterbankFeatures:
|
||||
weights *= enorm[:, np.newaxis]
|
||||
return weights
|
||||
|
||||
# STFT preperation
|
||||
# STFT preparation
|
||||
def pad_window_center(self, data, size, axis=-1, **kwargs):
|
||||
'''
|
||||
Centers the data and pads.
|
||||
@@ -329,7 +329,7 @@ class FilterbankFeatures:
|
||||
then padded with zeros to match n_fft
|
||||
fft_window : a vector or array of length `n_fft` having values computed by a
|
||||
window function
|
||||
pad_mode : mode while padding the singnal
|
||||
pad_mode : mode while padding the signal
|
||||
return_complex : returns array with complex data type if `True`
|
||||
return : Matrix of short-term Fourier transform coefficients.
|
||||
'''
|
||||
|
||||
@@ -122,28 +122,38 @@ class Mat:
|
||||
(dtype, ctype) = flags.dtype()
|
||||
elsize = np.dtype(dtype).itemsize
|
||||
|
||||
ptr = m['data']
|
||||
dataptr = int(ptr)
|
||||
length = (int(m['dataend']) - dataptr) // elsize
|
||||
start = (int(m['datastart']) - dataptr) // elsize
|
||||
shape = size.to_numpy()
|
||||
steps = np.asarray([int(m['step']['p'][i]) for i in range(len(shape))], dtype=np.int64)
|
||||
|
||||
if length == 0:
|
||||
ptr = m['data']
|
||||
# either we are default-constructed or sizes are zero
|
||||
if int(ptr) == 0 or np.prod(shape * steps) == 0:
|
||||
self.mat = np.array([])
|
||||
self.view = self.mat
|
||||
return
|
||||
|
||||
# we don't want to show excess brackets
|
||||
if flags.channels() != 1:
|
||||
shape = np.append(shape, flags.channels())
|
||||
steps = np.append(steps, elsize)
|
||||
|
||||
# get the length of contiguous array from data to the last element of the matrix
|
||||
length = 1 + np.sum((shape - 1) * steps) // elsize
|
||||
|
||||
if dtype != np.float16:
|
||||
# read all elements into self.mat
|
||||
ctype = gdb.lookup_type(ctype)
|
||||
ptr = ptr.cast(ctype.array(length - 1).pointer()).dereference()
|
||||
self.mat = np.array([ptr[i] for i in range(length)], dtype=dtype)
|
||||
else:
|
||||
# read as uint16_t and then reinterpret the bytes as float16
|
||||
u16 = gdb.lookup_type('uint16_t')
|
||||
ptr = ptr.cast(u16.array(length - 1).pointer()).dereference()
|
||||
self.mat = np.array([ptr[i] for i in range(length)], dtype=np.uint16)
|
||||
self.mat = self.mat.view(np.float16)
|
||||
|
||||
steps = np.asarray([int(m['step']['p'][i]) for i in range(size.dims())], dtype=np.int64)
|
||||
self.view = np.lib.stride_tricks.as_strided(self.mat[start:], shape=size.to_numpy(), strides=steps)
|
||||
# numpy will do the heavy lifting of strided access
|
||||
self.view = np.lib.stride_tricks.as_strided(self.mat, shape=shape, strides=steps)
|
||||
|
||||
def __iter__(self):
|
||||
return iter({'data': stri(self.view)}.items())
|
||||
|
||||
@@ -324,7 +324,10 @@ def main():
|
||||
is_work_scale_set = True
|
||||
img = cv.resize(src=full_img, dsize=None, fx=work_scale, fy=work_scale, interpolation=cv.INTER_LINEAR_EXACT)
|
||||
if is_seam_scale_set is False:
|
||||
seam_scale = min(1.0, np.sqrt(seam_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1])))
|
||||
if seam_megapix > 0:
|
||||
seam_scale = min(1.0, np.sqrt(seam_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1])))
|
||||
else:
|
||||
seam_scale = 1.0
|
||||
seam_work_aspect = seam_scale / work_scale
|
||||
is_seam_scale_set = True
|
||||
img_feat = cv.detail.computeImageFeatures2(finder, img)
|
||||
@@ -345,9 +348,9 @@ def main():
|
||||
img_names_subset = []
|
||||
full_img_sizes_subset = []
|
||||
for i in range(len(indices)):
|
||||
img_names_subset.append(img_names[indices[i, 0]])
|
||||
img_subset.append(images[indices[i, 0]])
|
||||
full_img_sizes_subset.append(full_img_sizes[indices[i, 0]])
|
||||
img_names_subset.append(img_names[indices[i]])
|
||||
img_subset.append(images[indices[i]])
|
||||
full_img_sizes_subset.append(full_img_sizes[indices[i]])
|
||||
images = img_subset
|
||||
img_names = img_names_subset
|
||||
full_img_sizes = full_img_sizes_subset
|
||||
@@ -479,7 +482,7 @@ def main():
|
||||
blender = cv.detail.Blender_createDefault(cv.detail.Blender_NO)
|
||||
elif blend_type == "multiband":
|
||||
blender = cv.detail_MultiBandBlender()
|
||||
blender.setNumBands((np.log(blend_width) / np.log(2.) - 1.).astype(np.int))
|
||||
blender.setNumBands((np.log(blend_width) / np.log(2.) - 1.).astype(np.int32))
|
||||
elif blend_type == "feather":
|
||||
blender = cv.detail_FeatherBlender()
|
||||
blender.setSharpness(1. / blend_width)
|
||||
@@ -513,6 +516,5 @@ def main():
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(__doc__)
|
||||
main()
|
||||
cv.destroyAllWindows()
|
||||
|
||||
Reference in New Issue
Block a user