diff --git a/app/src/main/cpp/MixingPlayer.h b/app/src/main/cpp/MixingPlayer.h index 77a78c7..56c8b87 100644 --- a/app/src/main/cpp/MixingPlayer.h +++ b/app/src/main/cpp/MixingPlayer.h @@ -56,6 +56,7 @@ public: } int getRate() { return kSampleRate; } + int getNumChannels() { return kChannelCount; } // Call this from Activity onPause() void stopAudio() { diff --git a/app/src/main/cpp/PlaybackEngine.cpp b/app/src/main/cpp/PlaybackEngine.cpp index f894bf9..abec43b 100644 --- a/app/src/main/cpp/PlaybackEngine.cpp +++ b/app/src/main/cpp/PlaybackEngine.cpp @@ -83,7 +83,9 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): android_fd(0), haveTimeRatio(false), timeRatio(1.0), - playbackRate(48000) + playbackRate(48000), + numOutChannels(2), + numInChannels(2) { LOGI("PlaybackEngine()"); LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL); @@ -93,8 +95,9 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): LOGI("read_mp3() is_ok=%d", is_ok); mPlayer = new MixingPlayer(samples); int32_t res = mPlayer->startAudio(); - playbackRate = mPlayer->getRate(); - LOGI("startAudio() = %d", res); + playbackRate.store(mPlayer->getRate()); + numOutChannels.store(mPlayer->getNumChannels()); + LOGI("startAudio() = %d rate=%d channels=%d", res, playbackRate.load(), numOutChannels.load()); initRubberBand(); } @@ -151,15 +154,53 @@ void PlaybackEngine::closeMusicFile() { } } +void PlaybackEngine::mapChannels(int *channel_map, int num_ch_in, int num_ch_out) { + if(num_ch_in == num_ch_out) { + // map each channel as-is + for(int i = 0; i < num_ch_out; i++) + channel_map[i] = i; + } else if(num_ch_in == 1) { + // map mono to all output channels + for(int i = 0; i < num_ch_out; i++) + channel_map[i] = 0; + } else if(num_ch_in >= 2) { + // use a stereo mapping + for(int i = 0; i < num_ch_out; i++) + channel_map[i] = i; + } else { + LOGE("mapChannels(): strange channel layout, mapping to mono. num_ch_in=%d num_ch_out=%d", num_ch_in, num_ch_out); + // map mono to all output channels + for(int i = 0; i < num_ch_out; i++) + channel_map[i] = 0; + } + // TODO: check broken input (0 channels etc) and bubble up an error to app +} + void PlaybackEngine::musicFeedThread() { + // strecher num channels: same as output num channels + // (this is because we play silence even without any input file, so we cannot set stretcher + // channel count for the music file's channel count) + int num_ch_in = numInChannels.load(); + int num_ch_out = numOutChannels.load(); + size_t num_buf_samples = buf_size_samples; size_t buf_stride = num_buf_samples; - float* buf = (float*) malloc(num_buf_samples * 2 * sizeof(float)); - float* buf_ptr[] {buf, buf + num_buf_samples}; - memset(buf, 0, num_buf_samples * 2 * sizeof(float)); - unsigned char* cbuf = (unsigned char*) malloc(num_buf_samples * 2 * sizeof(int16_t)); - memset(cbuf, 0, num_buf_samples * 2 * sizeof(int16_t)); - size_t cbuf_size_bytes = num_buf_samples * 2 * sizeof(int16_t); + size_t buf_size_bytes = num_buf_samples * num_ch_out * sizeof(float); + float* buf = (float*) malloc(buf_size_bytes); + float** buf_ptr = (float**) malloc(num_ch_out * sizeof(float*)); + for(int i = 0; i < num_ch_out; i++) { + buf_ptr[i] = buf + i * num_buf_samples; + } + memset(buf, 0, buf_size_bytes); + // preliminary allocation (actual music file buffer is unknown due to unknown channel count) + size_t cbuf_size_bytes = num_buf_samples * num_ch_in * sizeof(int16_t); + //size_t cbuf_load_bytes = num_buf_samples * num_ch_in * sizeof(int16_t); + unsigned char* cbuf = (unsigned char*) malloc(cbuf_size_bytes); + memset(cbuf, 0, cbuf_size_bytes); + + int* channel_map = (int*) malloc(num_ch_out * sizeof(int)); + + int loop_delay_ms = 1000 * buf_size_samples / playbackRate.load(); int idebug = 0; @@ -167,28 +208,38 @@ void PlaybackEngine::musicFeedThread() { while(!exitMusicFeedThread.load()) { if(haveTimeRatio.load()) { - stretcher.setTimeRatio(timeRatio); + stretcher.setTimeRatio(timeRatio.load()); haveTimeRatio.store(false); } + // change buffer size, if necessary (changed input channel count) + if(numInChannels.load() != num_ch_in) { + LOGD("changed buffer size (changed input channel count)"); + num_ch_in = numInChannels.load(); + free(cbuf); + cbuf_size_bytes = num_buf_samples * num_ch_in * sizeof(int16_t); + cbuf = (unsigned char*) malloc(cbuf_size_bytes); + memset(cbuf, 0, cbuf_size_bytes); + } + + mapChannels(channel_map, num_ch_in, num_ch_out); + // do work ... size_t num_samples = stretcher.getSamplesRequired(); // note: how much to sleep until output has played x samples...? - // can we just measure the wall time here, instead of calculating? -- that will be imprecise. // how large is one buffer, and when do we feed it more data? // (is it like double-buffering implemented in 'stretcher'?) - // can we just wait some bogus interval here, for a first version? if (num_samples == 0) { //LOGD("waiting for getSamplesRequired()"); - std::this_thread::sleep_for(std::chrono::milliseconds(20)); + std::this_thread::sleep_for(std::chrono::milliseconds(loop_delay_ms)); continue; } if (num_samples > num_buf_samples) { LOGE("wanted %d samples but buf is only %d samples", num_samples, num_buf_samples); - continue; + num_samples = num_buf_samples; } if (!haveMusicFile.load()) { @@ -197,7 +248,7 @@ void PlaybackEngine::musicFeedThread() { // 1024, 512, 512 // 7 x 512 } - memset(buf, 0, num_samples*2*sizeof(float)); + memset(buf, 0, num_samples*num_ch_out*sizeof(float)); stretcher.process(buf_ptr, num_samples, false); continue; } @@ -212,7 +263,7 @@ void PlaybackEngine::musicFeedThread() { size_t done = 0; // bytes! int err = mpg123_read(musicFile->handle, cbuf, cbuf_size_bytes, &done); musicFile->remaining_samples -= done / sizeof(int16_t); - musicFile->offset = 0; + musicFile->offset = 0; // unused here if (err != MPG123_OK && err != MPG123_DONE) { // error! LOGE("mpg123_read() err=%d done=%d", err, done); @@ -226,19 +277,25 @@ void PlaybackEngine::musicFeedThread() { continue; } - size_t num_decoded_samples = done / sizeof(int16_t) / 2; // 2 channels - TODO: actually use mp3 channels!! below, too. 2. + size_t num_decoded_samples = done / sizeof(int16_t) / num_ch_in; //LOGD("num_decoded_samples = %d", num_decoded_samples); - // convert interleaved int16 to de-interleaved float [-1.0, 1.0] format + // * convert interleaved int16 to de-interleaved float [-1.0, 1.0] format + // * map input to output channels for(size_t i = 0; i < num_decoded_samples; i++) { - for(size_t j = 0; j < 2; j++) { - buf[i + buf_stride * j] = static_cast(*(reinterpret_cast(cbuf) + i*2 + j)) / 32768.0f; + for(size_t j = 0; j < num_ch_out; j++) { + buf[i + buf_stride * j] = static_cast(*(reinterpret_cast(cbuf) + i * num_ch_in + channel_map[j])) / 32768.0f; } } //LOGD("calling stretcher.process()"); stretcher.process(buf_ptr, num_decoded_samples, false); } + + free(buf); + free(buf_ptr); + free(cbuf); + free(channel_map); } PlaybackEngine::~PlaybackEngine() { @@ -259,24 +316,30 @@ void PlaybackEngine::playMusic(int fd) { android_fd = fd; musicFile.reset(mp3file_open_fd(android_fd, 0)); if(musicFile) { - timeRatio = ((double) playbackRate) / ((double) musicFile->rate); + timeRatio.store(((double) playbackRate.load()) / ((double) musicFile->rate)); haveTimeRatio.store(true); + numInChannels.store(musicFile->channels); + haveMusicFile.store(true); } - haveMusicFile.store(true); - mPlayer->setMusic(std::make_shared(&stretcher, buf_size_samples)); + mPlayer->setMusic(std::make_shared(&stretcher, buf_size_samples, numOutChannels.load())); } -MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples) : +MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples, int num_ch_out) : stretcher(stretcher), idebug(0), - buf_size_samples(buf_size_samples) + buf_size_samples(buf_size_samples), + num_ch_out(num_ch_out) { - buf = (float*) malloc(buf_size_samples*2*sizeof(float)); - //float* buf_ptr[] {buf, buf + num_pad}; + buf = (float*) malloc(buf_size_samples*num_ch_out*sizeof(float)); + buf_ptr = (float**) malloc(num_ch_out * sizeof(float*)); + for(int i = 0; i < num_ch_out; i++) { + buf_ptr[i] = buf + i * buf_size_samples; + } } MusicProvider::~MusicProvider() { free(buf); + free(buf_ptr); } void MusicProvider::onAudioReady(float *data, int32_t frames) { @@ -285,18 +348,21 @@ void MusicProvider::onAudioReady(float *data, int32_t frames) { // frames=96 (48 kHz => 2 ms!!) } + if(frames > buf_size_samples) { + LOGE("MusicProvider::onAudioReady() asked for frames=%d but buf_size=%d", frames, buf_size_samples); + } + // 1. read from oboe into our temp de-interleaved buffer 'buf' size_t num_frames = std::min((size_t) frames, buf_size_samples); - float* buf_ptr[] {buf, buf + buf_size_samples}; stretcher->retrieve(buf_ptr, num_frames); // 2. convert to add samples to interleaved *data for(size_t i = 0; i < num_frames; i++) { - for(size_t j = 0; j < 2; j++) { - float sample = data[i*2 + j]; + for(size_t j = 0; j < num_ch_out; j++) { + float sample = data[i*num_ch_out + j]; sample += buf_ptr[j][i]; sample /= 2.0; - data[i*2 + j] = sample; + data[i*num_ch_out + j] = sample; } } } diff --git a/app/src/main/cpp/PlaybackEngine.h b/app/src/main/cpp/PlaybackEngine.h index 093f804..39daeb9 100644 --- a/app/src/main/cpp/PlaybackEngine.h +++ b/app/src/main/cpp/PlaybackEngine.h @@ -18,7 +18,7 @@ /** Provides music through a regular callback to oboe. Called from separate oboe thread. */ class MusicProvider : public AudioCallbackProvider { public: - explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples); + explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples, int num_ch_out); ~MusicProvider() override; /** Called from separate oboe thread. */ @@ -26,8 +26,10 @@ public: private: RubberBand::RubberBandStretcher *stretcher; float *buf; + float **buf_ptr; int idebug; size_t buf_size_samples; + int num_ch_out; }; class PlaybackEngine : public StepListener { @@ -47,13 +49,17 @@ private: std::atomic exitMusicFeedThread; int android_fd; std::atomic haveTimeRatio; - double timeRatio; - int playbackRate; + std::atomic timeRatio; + std::atomic playbackRate; + std::atomic numOutChannels; + std::atomic numInChannels; + /** this is actually in frames, not samples */ static size_t constexpr buf_size_samples = 1024; void initRubberBand(); void closeRubberBand(); void closeMusicFile(); void musicFeedThread(); + void mapChannels(int *channel_map, int num_ch_in, int num_ch_out); }; #endif //LOCKSTEP_PLAYBACKENGINE_H diff --git a/app/src/main/cpp/mp3file.h b/app/src/main/cpp/mp3file.h index 03e4419..e6a22b1 100644 --- a/app/src/main/cpp/mp3file.h +++ b/app/src/main/cpp/mp3file.h @@ -13,6 +13,7 @@ struct MP3File int android_fd; int channels; long rate; + /** num samples in total (stereo of 10 frames will have 20 'samples' here) */ long num_samples; int samples_per_frame; double secs_per_frame; @@ -20,6 +21,7 @@ struct MP3File double duration; size_t buffer_size; unsigned char* buffer; + /** total samples (stereo of 10 frames remaining will have 20 'remaining_samples' here) */ int remaining_samples; size_t offset; };