From b14ea02694ae75fcfce952759fc2cd11a08c9100 Mon Sep 17 00:00:00 2001 From: David Madl Date: Fri, 20 Mar 2026 23:17:34 +0100 Subject: [PATCH] feat: play music through librubberband --- TODO.md | 22 ++ app/src/main/cpp/AudioCallback.h | 20 ++ app/src/main/cpp/CMakeLists.txt | 1 + app/src/main/cpp/MixingPlayer.h | 21 +- app/src/main/cpp/PlaybackEngine.cpp | 220 +++++++++++++++++- app/src/main/cpp/PlaybackEngine.h | 31 +++ app/src/main/cpp/mp3file.cpp | 2 +- .../at/lockstep/app/LstForegroundService.java | 4 +- 8 files changed, 314 insertions(+), 7 deletions(-) create mode 100644 app/src/main/cpp/AudioCallback.h diff --git a/TODO.md b/TODO.md index 3e116c3..a921bb8 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,20 @@ ## TODO +* analyze the (secondly or so) noise beeps in the mp3 playback + - introduced with this commit + - is it librubberband, my failure to feed it properly (buffer exhaustion), or sth else? + +* correct sampling rate of libmpg123 vs. 48000 Hz using librubberband + +* record accelero and write recording to file + +* set feedForDelay in musicFeedThread() in PlaybackEngine.cpp + - LOGI("feed %d silence samples", num_samples); + - set the sleep delay + - ideally, set the buf size +* check playback buf size, and reduce buffer sizes + - LOGI("onAudioReady() frames=%d", frames); + * reduce lib size, librubberband is 1.3 M (one .so file) - maybe we are compiling too many source files - # TODO: see Android.mk in librubberband and copy options from `LOCAL_CFLAGS` @@ -26,6 +41,13 @@ O> 16 KB paging for NDK libs MixingPlayer currently forces both to 48000 and 2 respectively, regardless of what Android says would be optimal. +* nice-to nice-to: ramp up audio pipelines twice - once to collect params, another one to use those + +## Technical Debt + +* remove duplication of `mp3file_open()` in `mp3file_open_fd()` +* malloc() nullptr result handling + ## Before release * check librubberband license diff --git a/app/src/main/cpp/AudioCallback.h b/app/src/main/cpp/AudioCallback.h new file mode 100644 index 0000000..00dcda3 --- /dev/null +++ b/app/src/main/cpp/AudioCallback.h @@ -0,0 +1,20 @@ +// +// Created by david on 20.03.2026. +// + +#ifndef LOCKSTEP_AUDIOCALLBACK_H +#define LOCKSTEP_AUDIOCALLBACK_H + +#include + +/** + * Provides audio through a regular callback to oboe. + */ +class AudioCallbackProvider { +public: + virtual ~AudioCallbackProvider() {} + /** in current impl, this passes a buffer where data may already live. the provider may add to it and re-normalize to [-1.0, 1.0]. */ + virtual void onAudioReady(float *data, int32_t frames) {} +}; + +#endif //LOCKSTEP_AUDIOCALLBACK_H diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt index 67924ef..62815a2 100644 --- a/app/src/main/cpp/CMakeLists.txt +++ b/app/src/main/cpp/CMakeLists.txt @@ -51,6 +51,7 @@ set_target_properties(mpg123 PROPERTIES IMPORTED_LOCATION include_directories(${mpg123_DIR}/lib/${ANDROID_ABI}/include) target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE libpasada/pasada-lib/include) +target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE librubberband/rubberband) # Specifies libraries CMake should link to your target library. You # can link libraries from various origins, such as libraries defined in this diff --git a/app/src/main/cpp/MixingPlayer.h b/app/src/main/cpp/MixingPlayer.h index 4001295..29bc6b3 100644 --- a/app/src/main/cpp/MixingPlayer.h +++ b/app/src/main/cpp/MixingPlayer.h @@ -6,7 +6,10 @@ #define LOCKSTEP_MIXINGPLAYER_H #include -#include +#include +#include +#include "AudioCallback.h" + using namespace oboe; /** @@ -20,7 +23,7 @@ protected: std::atomic startBeat; int numBeatsPlaying; public: - explicit MixingPlayer(std::vector beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0) {} + explicit MixingPlayer(std::vector beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0), mHaveMusic(false) {} virtual ~MixingPlayer() = default; @@ -63,6 +66,12 @@ public: } } + void setMusic(std::shared_ptr cb) { + std::lock_guard lock(mLock); + mMusic = std::move(cb); + mHaveMusic.store((bool) mMusic); + } + oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override { // fetch startBeat register int isStartBeat = startBeat.load(); @@ -78,7 +87,7 @@ public: int N = (int) beatSound.size(); for (int i = 0; i < numFrames; i++) { float sample = 0.0; - float norm = (float) numBeatsPlaying; + float norm = (float) (std::max(numBeatsPlaying, 1)); for (int k = 0; k < numBeatsPlaying; k++) { sample += beatSound[beatIdx[k]]; beatIdx[k] += 1; @@ -102,12 +111,18 @@ public: } } + if(mHaveMusic.load()) { + mMusic->onAudioReady(floatData, numFrames); + } + return oboe::DataCallbackResult::Continue; } private: std::mutex mLock; std::shared_ptr mStream; + std::shared_ptr mMusic; + std::atomic mHaveMusic; // Stream params static int constexpr kChannelCount = 2; diff --git a/app/src/main/cpp/PlaybackEngine.cpp b/app/src/main/cpp/PlaybackEngine.cpp index afc60e4..134b04f 100644 --- a/app/src/main/cpp/PlaybackEngine.cpp +++ b/app/src/main/cpp/PlaybackEngine.cpp @@ -8,7 +8,9 @@ #include "logging.h" #include "mpg123.h" #include "mp3file.h" +#include #include +#include /** * Read samples from the next mp3-frame into the struct MP3File's buffer. @@ -52,7 +54,34 @@ static bool read_mp3(std::string filename, std::vector& samples) { return ok1 && ok2; } -PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(filesDir) { + +struct RbLogger : public RubberBand::RubberBandStretcher::Logger { + virtual void log(const char *s) { + LOGI("%s", s); + } + virtual void log(const char *s, double val) { + LOGI("%s (%lf)", s, val); + } + virtual void log(const char *s, double val1, double val2) { + LOGI("%s (%lf, %lf)", s, val1, val2); + } + virtual ~RbLogger() { } +}; + +PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): + stretcher( + /* sampleRate: */ 48000, + /* channels: */ 2, + /* logger: */ std::make_shared(), + /* options: */ + RubberBand::RubberBandStretcher::OptionEngineFaster | + RubberBand::RubberBandStretcher::OptionProcessRealTime + ), + mFilesDir(filesDir), + haveMusicFile(false), + exitMusicFeedThread(false), + android_fd(0) +{ LOGI("PlaybackEngine()"); LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL); // NDK LOG_LEVEL=3 (DEBUG) @@ -62,9 +91,148 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(files mPlayer = new MixingPlayer(samples); int32_t res = mPlayer->startAudio(); LOGI("startAudio() = %d", res); + initRubberBand(); +} + +void PlaybackEngine::initRubberBand() { + // TODO: check mp3 actual sample rate, and adapt to 48000 Hz playback here + stretcher.setTimeRatio(1.0); + stretcher.setPitchScale(1.0); + stretcher.setDebugLevel(1); // 1: errors only. generally 0..4 + + // feed samples into 'stretcher' and read bogus output + // getPreferredStartPad() -> [ ... ] -> getStartDelay() + + // write silence + // process() input: de-interleaved audio data with one float array per channel. + size_t num_pad = stretcher.getPreferredStartPad(); + float* buf = (float*) malloc(num_pad*2*sizeof(float)); + float* buf_ptr[] {buf, buf + num_pad}; + memset(buf, 0, num_pad*2*sizeof(float)); + stretcher.process(buf_ptr, num_pad, false); + free(buf); + //LOGI("start_pad = %d", num_pad); + + // read bogus output + size_t start_delay = stretcher.getStartDelay(); + float* out_buf = (float*) malloc(start_delay*2*sizeof(float)); + float* out_buf_ptr[] {out_buf, out_buf + start_delay}; + memset(out_buf, 0, start_delay*2*sizeof(float)); + stretcher.retrieve(out_buf_ptr, start_delay); + //LOGI("start_delay = %d", start_delay); + + // thread 1: oboe cb fetching via retrieve() + //mPlayer->setMusic(); + + // thread 2: polling for decoding more mp3 -> process() -- getSamplesRequired() + // setTimeRatio(), setPitchScale() -- always call them from thread 2 + musicFeed = std::make_unique(&PlaybackEngine::musicFeedThread, this); +} + +void PlaybackEngine::closeRubberBand() { + if(musicFeed) { + exitMusicFeedThread.store(true); + musicFeed->join(); + musicFeed = nullptr; + } + closeMusicFile(); +} + +void PlaybackEngine::closeMusicFile() { + haveMusicFile.store(false); + musicFile = nullptr; + if(android_fd) { + close(android_fd); + android_fd = 0; + } +} + +void PlaybackEngine::musicFeedThread() { + // refactor: rename to 'num_buf_samples' + size_t num_pad = 48000; // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()? + size_t buf_stride = num_pad; + float* buf = (float*) malloc(num_pad*2*sizeof(float)); + float* buf_ptr[] {buf, buf + num_pad}; + memset(buf, 0, num_pad*2*sizeof(float)); + unsigned char* cbuf = (unsigned char*) malloc(num_pad*2*sizeof(int16_t)); + memset(cbuf, 0, num_pad*2*sizeof(int16_t)); + size_t cbuf_size_bytes = num_pad*2*sizeof(int16_t); + + int idebug = 0; + + // thread 2: polling for decoding more mp3 -> process() -- getSamplesRequired() + while(!exitMusicFeedThread.load()) { + // do work ... + size_t num_samples = stretcher.getSamplesRequired(); + + // note: how much to sleep until output has played x samples...? + // can we just measure the wall time here, instead of calculating? -- that will be imprecise. + // how large is one buffer, and when do we feed it more data? + // (is it like double-buffering implemented in 'stretcher'?) + + // can we just wait some bogus interval here, for a first version? + if (num_samples == 0) { + //LOGD("waiting for getSamplesRequired()"); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + continue; + } + + if (num_samples > num_pad) { + LOGE("wanted %d samples but buf is only %d samples", num_samples, num_pad); + continue; + } + + if (!haveMusicFile.load()) { + if(idebug++ < 10) { + LOGI("feed %d silence samples", num_samples); + // 1024, 512, 512 + // 7 x 512 + } + memset(buf, 0, num_samples*2*sizeof(float)); + stretcher.process(buf_ptr, num_samples, false); + continue; + } + + if(idebug++ < 10) { + LOGI("feed %d music samples", num_samples); + // feed 1024 music samples + // => stretcher is asking for 1024 = getSamplesRequired() + // ca. 21 ms + } + + size_t done = 0; // bytes! + int err = mpg123_read(musicFile->handle, cbuf, cbuf_size_bytes, &done); + musicFile->remaining_samples -= done / sizeof(int16_t); + musicFile->offset = 0; + if (err != MPG123_OK && err != MPG123_DONE) { + // error! + LOGE("mpg123_read() err=%d done=%d", err, done); + // next iteration will play silence + closeMusicFile(); + continue; + } + if(err == MPG123_DONE) { + // next iteration will play silence + closeMusicFile(); + continue; + } + + size_t num_decoded_samples = done / sizeof(int16_t) / 2; // 2 channels - TODO: actually use mp3 channels!! below, too. 2. + LOGI("num_decoded_samples = %d", num_decoded_samples); + // convert interleaved int16 to de-interleaved float [-1.0, 1.0] format + for(size_t i = 0; i < num_decoded_samples; i++) { + for(size_t j = 0; j < 2; j++) { + buf[i + buf_stride * j] = static_cast(*(reinterpret_cast(cbuf) + i*2 + j)) / 32768.0f; + } + } + + LOGI("calling stretcher.process()"); + stretcher.process(buf_ptr, num_decoded_samples, false); + } } PlaybackEngine::~PlaybackEngine() { + closeRubberBand(); LOGI("~PlaybackEngine()"); mPlayer->stopAudio(); delete mPlayer; @@ -76,9 +244,57 @@ void PlaybackEngine::playBeat() { } void PlaybackEngine::playMusic(int fd) { + if(!mPlayer) return; + // TODO: fetch sampling rate from mp3 file, and use librubberband to correct for it + // MixingPlayer::kSampleRate (48000) + // mp3->rate + + // feed samples to librubberband + // fetch resamples out of librubberband + //if(mPlayer) mPlayer->playMusic(); // TODO: fd is opened; dispose of fd when stopping or being discarded ... LOGI("PlaybackEngine::playMusic(fd=%d)", fd); - close(fd); // for now, nothing is implemented. we just close it again. + //close(fd); // for now, nothing is implemented. we just close it again. // we will use mp3file_open_fd() later. + + android_fd = fd; + musicFile.reset(mp3file_open_fd(android_fd, 0)); + haveMusicFile.store(true); + mPlayer->setMusic(std::make_shared(&stretcher)); +} + +MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher) : stretcher(stretcher), idebug(0) { + // refactor: rename to 'num_buf_samples' + // TODO: for cache-friendliness, it would be better to have smaller 'num_buf_samples' + // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()? + //size_t buf_stride = num_pad; + buf = (float*) malloc(num_buf_samples*2*sizeof(float)); + //float* buf_ptr[] {buf, buf + num_pad}; +} + +MusicProvider::~MusicProvider() { + free(buf); +} + +void MusicProvider::onAudioReady(float *data, int32_t frames) { + if(idebug++ < 10) { + LOGI("onAudioReady() frames=%d", frames); + // frames=96 (48 kHz => 2 ms!!) + } + + // 1. read from oboe into our temp de-interleaved buffer 'buf' + size_t num_frames = std::min((size_t) frames, num_buf_samples); + float* buf_ptr[] {buf, buf + num_buf_samples}; + stretcher->retrieve(buf_ptr, num_frames); + + // 2. convert to add samples to interleaved *data + for(size_t i = 0; i < num_frames; i++) { + for(size_t j = 0; j < 2; j++) { + float sample = data[i*2 + j]; + sample += buf_ptr[j][i]; + sample /= 2.0; + data[i*2 + j] = sample; + } + } } diff --git a/app/src/main/cpp/PlaybackEngine.h b/app/src/main/cpp/PlaybackEngine.h index 4520add..9546326 100644 --- a/app/src/main/cpp/PlaybackEngine.h +++ b/app/src/main/cpp/PlaybackEngine.h @@ -7,7 +7,28 @@ #include "StepListener.h" #include "MixingPlayer.h" +#include "RubberBandStretcher.h" +#include "mp3file.h" +#include "AudioCallback.h" #include +#include +#include +#include + +/** Provides music through a regular callback to oboe. Called from separate oboe thread. */ +class MusicProvider : public AudioCallbackProvider { +public: + explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher); + ~MusicProvider() override; + + /** Called from separate oboe thread. */ + void onAudioReady(float *data, int32_t frames) override; +private: + const size_t num_buf_samples = 48000; + RubberBand::RubberBandStretcher *stretcher; + float *buf; + int idebug; +}; class PlaybackEngine : public StepListener { public: @@ -17,8 +38,18 @@ public: virtual void playBeat(); void playMusic(int fd); private: + RubberBand::RubberBandStretcher stretcher; MixingPlayer *mPlayer; std::string mFilesDir; + std::unique_ptr musicFile; + std::atomic haveMusicFile; + std::unique_ptr musicFeed; + std::atomic exitMusicFeedThread; + int android_fd; + void initRubberBand(); + void closeRubberBand(); + void closeMusicFile(); + void musicFeedThread(); }; #endif //LOCKSTEP_PLAYBACKENGINE_H diff --git a/app/src/main/cpp/mp3file.cpp b/app/src/main/cpp/mp3file.cpp index f5c16bb..fb0f269 100644 --- a/app/src/main/cpp/mp3file.cpp +++ b/app/src/main/cpp/mp3file.cpp @@ -151,7 +151,7 @@ MP3File* mp3file_open_fd(int fd, int forceEncoding) { mp3->offset = 0; mp3->remaining_samples = (int) mp3->num_samples; - LOGV("channels: %d rate: %ld num_samples: %ld", mp3->channels, mp3->rate, mp3->num_samples); + LOGI("channels: %d rate: %ld num_samples: %ld", mp3->channels, mp3->rate, mp3->num_samples); mp3->android_fd = fd; return mp3; diff --git a/app/src/main/java/at/lockstep/app/LstForegroundService.java b/app/src/main/java/at/lockstep/app/LstForegroundService.java index 73b24f6..fb00eed 100644 --- a/app/src/main/java/at/lockstep/app/LstForegroundService.java +++ b/app/src/main/java/at/lockstep/app/LstForegroundService.java @@ -87,7 +87,9 @@ public class LstForegroundService extends Service implements SensorEventListener if (ACTION_START.equals(action)) { String contentUri = intent.getStringExtra("content_uri"); try { - PlaybackEngine.playMusic(uriToFd(contentUri)); + if(contentUri != null) { + PlaybackEngine.playMusic(uriToFd(contentUri)); + } } catch (IOException e) { // TODO proper error handling Toast.makeText(this, "Could not open music file contentUri", Toast.LENGTH_LONG).show();