1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include <stdio.h>
16 #include "micro_speech/feature_provider.h"
17
18 #include "micro_speech/audio_provider.h"
19 #include "micro_speech/micro_features/micro_features_generator.h"
20 #include "micro_speech/micro_features/micro_model_settings.h"
21
FeatureProvider(int feature_size,int8_t * feature_data)22 FeatureProvider::FeatureProvider(int feature_size, int8_t* feature_data)
23 : feature_size_(feature_size),
24 feature_data_(feature_data),
25 is_first_run_(true) {
26 // Initialize the feature data to default values.
27 for (int n = 0; n < feature_size_; ++n) {
28 feature_data_[n] = 0;
29 }
30 }
31
~FeatureProvider()32 FeatureProvider::~FeatureProvider() {}
33
PopulateFeatureData(tflite::ErrorReporter * error_reporter,int32_t last_time_in_ms,int32_t time_in_ms,int * how_many_new_slices)34 TfLiteStatus FeatureProvider::PopulateFeatureData(
35 tflite::ErrorReporter* error_reporter, int32_t last_time_in_ms,
36 int32_t time_in_ms, int* how_many_new_slices) {
37 if (feature_size_ != kFeatureElementCount) {
38 TF_LITE_REPORT_ERROR(error_reporter,
39 "Requested feature_data_ size %d doesn't match %d",
40 feature_size_, kFeatureElementCount);
41 printf("Requested feature_data_ size %d doesn't match %d",
42 feature_size_, kFeatureElementCount);
43 return kTfLiteError;
44 }
45
46 // Quantize the time into steps as long as each window stride, so we can
47 // figure out which audio data we need to fetch.
48 const int last_step = (last_time_in_ms / kFeatureSliceStrideMs);
49 const int current_step = (time_in_ms / kFeatureSliceStrideMs);
50
51 int slices_needed = current_step - last_step;
52 //printf("[lk added]slices_needed = (current_step : %d - last_step: %d) = %d\n", current_step, last_step, slices_needed);
53 //printf("[lk added]slices_needed : %d\n", slices_needed);
54 // If this is the first call, make sure we don't use any cached information.
55 if (is_first_run_) {
56 TfLiteStatus init_status = InitializeMicroFeatures(error_reporter);
57 if (init_status != kTfLiteOk) {
58 printf("init_status: %d\n", init_status);
59 return init_status;
60 }
61 is_first_run_ = false;
62 slices_needed = kFeatureSliceCount;
63 }
64 if (slices_needed > kFeatureSliceCount) {
65 slices_needed = kFeatureSliceCount;
66 }
67 *how_many_new_slices = slices_needed;
68 //printf("slices_needed 2: %d\n", slices_needed);
69 const int slices_to_keep = kFeatureSliceCount - slices_needed;
70 const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
71 // If we can avoid recalculating some slices, just move the existing data
72 // up in the spectrogram, to perform something like this:
73 // last time = 80ms current time = 120ms
74 // +-----------+ +-----------+
75 // | data@20ms | --> | data@60ms |
76 // +-----------+ -- +-----------+
77 // | data@40ms | -- --> | data@80ms |
78 // +-----------+ -- -- +-----------+
79 // | data@60ms | -- -- | <empty> |
80 // +-----------+ -- +-----------+
81 // | data@80ms | -- | <empty> |
82 // +-----------+ +-----------+
83 if (slices_to_keep > 0) {
84 for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
85 int8_t* dest_slice_data =
86 feature_data_ + (dest_slice * kFeatureSliceSize);
87 const int src_slice = dest_slice + slices_to_drop;
88 const int8_t* src_slice_data =
89 feature_data_ + (src_slice * kFeatureSliceSize);
90 for (int i = 0; i < kFeatureSliceSize; ++i) {
91 dest_slice_data[i] = src_slice_data[i];
92 }
93 }
94 }
95 // Any slices that need to be filled in with feature data have their
96 // appropriate audio data pulled, and features calculated for that slice.
97 if (slices_needed > 0) {
98 for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount;
99 ++new_slice) {
100 const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
101 const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
102 int16_t* audio_samples = nullptr;
103 int audio_samples_size = 0;
104 // TODO(petewarden): Fix bug that leads to non-zero slice_start_ms
105 GetAudioSamples(error_reporter, (slice_start_ms > 0 ? slice_start_ms : 0),
106 kFeatureSliceDurationMs, &audio_samples_size,
107 &audio_samples);
108 if (audio_samples_size < kMaxAudioSampleSize) {
109 TF_LITE_REPORT_ERROR(error_reporter,
110 "Audio data size %d too small, want %d",
111 audio_samples_size, kMaxAudioSampleSize);
112 return kTfLiteError;
113 }
114 int8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
115 size_t num_samples_read;
116 TfLiteStatus generate_status = GenerateMicroFeatures(
117 error_reporter, audio_samples, audio_samples_size, kFeatureSliceSize,
118 new_slice_data, &num_samples_read);
119 if (generate_status != kTfLiteOk) {
120 return generate_status;
121 }
122 }
123 }
124 return kTfLiteOk;
125 }
126