feat: play music through librubberband

TODO.md

@@ -1,5 +1,20 @@
 ## TODO
 
+* analyze the (secondly or so) noise beeps in the mp3 playback
+  - introduced with this commit
+  - is it librubberband, my failure to feed it properly (buffer exhaustion), or sth else?
+
+* correct sampling rate of libmpg123 vs. 48000 Hz using librubberband
+
+* record accelero and write recording to file
+
+* set feedForDelay in musicFeedThread() in PlaybackEngine.cpp
+  -  LOGI("feed %d silence samples", num_samples);
+  - set the sleep delay
+  - ideally, set the buf size
+* check playback buf size, and reduce buffer sizes
+  - LOGI("onAudioReady() frames=%d", frames);
+
 * reduce lib size, librubberband is 1.3 M (one .so file)
     - maybe we are compiling too many source files
     - # TODO: see Android.mk in librubberband and copy options from `LOCAL_CFLAGS`
@@ -26,6 +41,13 @@ O> 16 KB paging for NDK libs
   MixingPlayer currently forces both to 48000 and 2 respectively,
   regardless of what Android says would be optimal.
 
+* nice-to nice-to: ramp up audio pipelines twice - once to collect params, another one to use those
+
+## Technical Debt
+
+* remove duplication of `mp3file_open()` in `mp3file_open_fd()`
+* malloc() nullptr result handling
+
 ## Before release
 
 * check librubberband license

app/src/main/cpp/AudioCallback.h

@@ -0,0 +1,20 @@
+//
+// Created by david on 20.03.2026.
+//
+
+#ifndef LOCKSTEP_AUDIOCALLBACK_H
+#define LOCKSTEP_AUDIOCALLBACK_H
+
+#include <cstdint>
+
+/**
+ * Provides audio through a regular callback to oboe.
+ */
+class AudioCallbackProvider {
+public:
+    virtual ~AudioCallbackProvider() {}
+    /** in current impl, this passes a buffer where data may already live. the provider may add to it and re-normalize to [-1.0, 1.0]. */
+    virtual void onAudioReady(float *data, int32_t frames) {}
+};
+
+#endif //LOCKSTEP_AUDIOCALLBACK_H

app/src/main/cpp/CMakeLists.txt

@@ -51,6 +51,7 @@ set_target_properties(mpg123 PROPERTIES IMPORTED_LOCATION
 include_directories(${mpg123_DIR}/lib/${ANDROID_ABI}/include)
 
 target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE libpasada/pasada-lib/include)
+target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE librubberband/rubberband)
 
 # Specifies libraries CMake should link to your target library. You
 # can link libraries from various origins, such as libraries defined in this

app/src/main/cpp/MixingPlayer.h

@@ -6,7 +6,10 @@
 #define LOCKSTEP_MIXINGPLAYER_H
 
 #include <oboe/Oboe.h>
-#include <math.h>
+#include <cmath>
+#include <atomic>
+#include "AudioCallback.h"
+
 using namespace oboe;
 
 /**
@@ -20,7 +23,7 @@ protected:
     std::atomic<int> startBeat;
     int numBeatsPlaying;
 public:
-    explicit MixingPlayer(std::vector<float> beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0) {}
+    explicit MixingPlayer(std::vector<float> beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0), mHaveMusic(false) {}
 
     virtual ~MixingPlayer() = default;
 
@@ -63,6 +66,12 @@ public:
         }
     }
 
+    void setMusic(std::shared_ptr<AudioCallbackProvider> cb) {
+        std::lock_guard<std::mutex> lock(mLock);
+        mMusic = std::move(cb);
+        mHaveMusic.store((bool) mMusic);
+    }
+
     oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override {
         // fetch startBeat register
         int isStartBeat = startBeat.load();
@@ -78,7 +87,7 @@ public:
         int N = (int) beatSound.size();
         for (int i = 0; i < numFrames; i++) {
             float sample = 0.0;
-            float norm = (float) numBeatsPlaying;
+            float norm = (float) (std::max(numBeatsPlaying, 1));
             for (int k = 0; k < numBeatsPlaying; k++) {
                 sample += beatSound[beatIdx[k]];
                 beatIdx[k] += 1;
@@ -102,12 +111,18 @@ public:
             }
         }
 
+        if(mHaveMusic.load()) {
+            mMusic->onAudioReady(floatData, numFrames);
+        }
+
         return oboe::DataCallbackResult::Continue;
     }
 
 private:
     std::mutex         mLock;
     std::shared_ptr<oboe::AudioStream> mStream;
+    std::shared_ptr<AudioCallbackProvider> mMusic;
+    std::atomic<bool> mHaveMusic;
 
     // Stream params
     static int constexpr kChannelCount = 2;

app/src/main/cpp/PlaybackEngine.cpp

@@ -8,7 +8,9 @@
 #include "logging.h"
 #include "mpg123.h"
 #include "mp3file.h"
+#include <memory>
 #include <vector>
+#include <chrono>
 
 /**
  * Read samples from the next mp3-frame into the struct MP3File's buffer.
@@ -52,7 +54,34 @@ static bool read_mp3(std::string filename, std::vector<float>& samples) {
     return ok1 && ok2;
 }
 
-PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(filesDir) {
+
+struct RbLogger : public RubberBand::RubberBandStretcher::Logger {
+    virtual void log(const char *s) {
+        LOGI("%s", s);
+    }
+    virtual void log(const char *s, double val) {
+        LOGI("%s (%lf)", s, val);
+    }
+    virtual void log(const char *s, double val1, double val2) {
+        LOGI("%s (%lf, %lf)", s, val1, val2);
+    }
+    virtual ~RbLogger() { }
+};
+
+PlaybackEngine::PlaybackEngine(std::string filesDir, int resid):
+    stretcher(
+            /* sampleRate: */ 48000,
+            /* channels: */ 2,
+            /* logger: */ std::make_shared<RbLogger>(),
+            /* options: */
+            RubberBand::RubberBandStretcher::OptionEngineFaster |
+            RubberBand::RubberBandStretcher::OptionProcessRealTime
+    ),
+    mFilesDir(filesDir),
+    haveMusicFile(false),
+    exitMusicFeedThread(false),
+    android_fd(0)
+{
     LOGI("PlaybackEngine()");
     LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL);
     // NDK LOG_LEVEL=3  (DEBUG)
@@ -62,9 +91,148 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(files
     mPlayer = new MixingPlayer(samples);
     int32_t res = mPlayer->startAudio();
     LOGI("startAudio() = %d", res);
+    initRubberBand();
+}
+
+void PlaybackEngine::initRubberBand() {
+    // TODO: check mp3 actual sample rate, and adapt to 48000 Hz playback here
+    stretcher.setTimeRatio(1.0);
+    stretcher.setPitchScale(1.0);
+    stretcher.setDebugLevel(1); // 1: errors only. generally 0..4
+
+    // feed samples into 'stretcher' and read bogus output
+    // getPreferredStartPad() -> [ ... ] -> getStartDelay()
+
+    // write silence
+    // process() input: de-interleaved audio data with one float array per channel.
+    size_t num_pad = stretcher.getPreferredStartPad();
+    float* buf = (float*) malloc(num_pad*2*sizeof(float));
+    float* buf_ptr[] {buf, buf + num_pad};
+    memset(buf, 0, num_pad*2*sizeof(float));
+    stretcher.process(buf_ptr, num_pad, false);
+    free(buf);
+    //LOGI("start_pad = %d", num_pad);
+
+    // read bogus output
+    size_t start_delay = stretcher.getStartDelay();
+    float* out_buf = (float*) malloc(start_delay*2*sizeof(float));
+    float* out_buf_ptr[] {out_buf, out_buf + start_delay};
+    memset(out_buf, 0, start_delay*2*sizeof(float));
+    stretcher.retrieve(out_buf_ptr, start_delay);
+    //LOGI("start_delay = %d", start_delay);
+
+    // thread 1: oboe cb fetching via retrieve()
+    //mPlayer->setMusic();
+
+    // thread 2: polling for decoding more mp3 -> process()  -- getSamplesRequired()
+    // setTimeRatio(), setPitchScale() -- always call them from thread 2
+    musicFeed = std::make_unique<std::thread>(&PlaybackEngine::musicFeedThread, this);
+}
+
+void PlaybackEngine::closeRubberBand() {
+    if(musicFeed) {
+        exitMusicFeedThread.store(true);
+        musicFeed->join();
+        musicFeed = nullptr;
+    }
+    closeMusicFile();
+}
+
+void PlaybackEngine::closeMusicFile() {
+    haveMusicFile.store(false);
+    musicFile = nullptr;
+    if(android_fd) {
+        close(android_fd);
+        android_fd = 0;
+    }
+}
+
+void PlaybackEngine::musicFeedThread() {
+    // refactor: rename to 'num_buf_samples'
+    size_t num_pad = 48000; // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()?
+    size_t buf_stride = num_pad;
+    float* buf = (float*) malloc(num_pad*2*sizeof(float));
+    float* buf_ptr[] {buf, buf + num_pad};
+    memset(buf, 0, num_pad*2*sizeof(float));
+    unsigned char* cbuf = (unsigned char*) malloc(num_pad*2*sizeof(int16_t));
+    memset(cbuf, 0, num_pad*2*sizeof(int16_t));
+    size_t cbuf_size_bytes = num_pad*2*sizeof(int16_t);
+
+    int idebug = 0;
+
+    // thread 2: polling for decoding more mp3 -> process()  -- getSamplesRequired()
+    while(!exitMusicFeedThread.load()) {
+        // do work ...
+        size_t num_samples = stretcher.getSamplesRequired();
+
+        // note: how much to sleep until output has played x samples...?
+        // can we just measure the wall time here, instead of calculating? -- that will be imprecise.
+        // how large is one buffer, and when do we feed it more data?
+        // (is it like double-buffering implemented in 'stretcher'?)
+
+        // can we just wait some bogus interval here, for a first version?
+        if (num_samples == 0) {
+            //LOGD("waiting for getSamplesRequired()");
+            std::this_thread::sleep_for(std::chrono::milliseconds(20));
+            continue;
+        }
+
+        if (num_samples > num_pad) {
+            LOGE("wanted %d samples but buf is only %d samples", num_samples, num_pad);
+            continue;
+        }
+
+        if (!haveMusicFile.load()) {
+            if(idebug++ < 10) {
+                LOGI("feed %d silence samples", num_samples);
+                // 1024, 512, 512
+                // 7 x 512
+            }
+            memset(buf, 0, num_samples*2*sizeof(float));
+            stretcher.process(buf_ptr, num_samples, false);
+            continue;
+        }
+
+        if(idebug++ < 10) {
+            LOGI("feed %d music samples", num_samples);
+            // feed 1024 music samples
+            // => stretcher is asking for 1024 = getSamplesRequired()
+            // ca. 21 ms
+        }
+
+        size_t done = 0; // bytes!
+        int err = mpg123_read(musicFile->handle, cbuf, cbuf_size_bytes, &done);
+        musicFile->remaining_samples -= done / sizeof(int16_t);
+        musicFile->offset = 0;
+        if (err != MPG123_OK && err != MPG123_DONE) {
+            // error!
+            LOGE("mpg123_read() err=%d done=%d", err, done);
+            // next iteration will play silence
+            closeMusicFile();
+            continue;
+        }
+        if(err == MPG123_DONE) {
+            // next iteration will play silence
+            closeMusicFile();
+            continue;
+        }
+
+        size_t num_decoded_samples = done / sizeof(int16_t) / 2; // 2 channels - TODO: actually use mp3 channels!! below, too. 2.
+        LOGI("num_decoded_samples = %d", num_decoded_samples);
+        // convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
+        for(size_t i = 0; i < num_decoded_samples; i++) {
+            for(size_t j = 0; j < 2; j++) {
+                buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i*2 + j)) / 32768.0f;
+            }
+        }
+
+        LOGI("calling stretcher.process()");
+        stretcher.process(buf_ptr, num_decoded_samples, false);
+    }
 }
 
 PlaybackEngine::~PlaybackEngine() {
+    closeRubberBand();
     LOGI("~PlaybackEngine()");
     mPlayer->stopAudio();
     delete mPlayer;
@@ -76,9 +244,57 @@ void PlaybackEngine::playBeat() {
 }
 
 void PlaybackEngine::playMusic(int fd) {
+    if(!mPlayer) return;
+    // TODO: fetch sampling rate from mp3 file, and use librubberband to correct for it
+    // MixingPlayer::kSampleRate (48000)
+    // mp3->rate
+
+    // feed samples to librubberband
+    // fetch resamples out of librubberband
+
     //if(mPlayer) mPlayer->playMusic();
     // TODO: fd is opened; dispose of fd when stopping or being discarded ...
     LOGI("PlaybackEngine::playMusic(fd=%d)", fd);
-    close(fd); // for now, nothing is implemented. we just close it again.
+    //close(fd); // for now, nothing is implemented. we just close it again.
     // we will use mp3file_open_fd() later.
+
+    android_fd = fd;
+    musicFile.reset(mp3file_open_fd(android_fd, 0));
+    haveMusicFile.store(true);
+    mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher));
+}
+
+MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher) : stretcher(stretcher), idebug(0) {
+    // refactor: rename to 'num_buf_samples'
+    // TODO: for cache-friendliness, it would be better to have smaller 'num_buf_samples'
+    // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()?
+    //size_t buf_stride = num_pad;
+    buf = (float*) malloc(num_buf_samples*2*sizeof(float));
+    //float* buf_ptr[] {buf, buf + num_pad};
+}
+
+MusicProvider::~MusicProvider() {
+    free(buf);
+}
+
+void MusicProvider::onAudioReady(float *data, int32_t frames) {
+    if(idebug++ < 10) {
+        LOGI("onAudioReady() frames=%d", frames);
+        // frames=96 (48 kHz => 2 ms!!)
+    }
+
+    // 1. read from oboe into our temp de-interleaved buffer 'buf'
+    size_t num_frames = std::min((size_t) frames, num_buf_samples);
+    float* buf_ptr[] {buf, buf + num_buf_samples};
+    stretcher->retrieve(buf_ptr, num_frames);
+
+    // 2. convert to add samples to interleaved *data
+    for(size_t i = 0; i < num_frames; i++) {
+        for(size_t j = 0; j < 2; j++) {
+            float sample = data[i*2 + j];
+            sample += buf_ptr[j][i];
+            sample /= 2.0;
+            data[i*2 + j] = sample;
+        }
+    }
 }

app/src/main/cpp/PlaybackEngine.h

@@ -7,7 +7,28 @@
 
 #include "StepListener.h"
 #include "MixingPlayer.h"
+#include "RubberBandStretcher.h"
+#include "mp3file.h"
+#include "AudioCallback.h"
 #include <string>
+#include <thread>
+#include <memory>
+#include <atomic>
+
+/** Provides music through a regular callback to oboe. Called from separate oboe thread. */
+class MusicProvider : public AudioCallbackProvider {
+public:
+    explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher);
+    ~MusicProvider() override;
+
+    /** Called from separate oboe thread. */
+    void onAudioReady(float *data, int32_t frames) override;
+private:
+    const size_t num_buf_samples = 48000;
+    RubberBand::RubberBandStretcher *stretcher;
+    float *buf;
+    int idebug;
+};
 
 class PlaybackEngine : public StepListener {
 public:
@@ -17,8 +38,18 @@ public:
     virtual void playBeat();
     void playMusic(int fd);
 private:
+    RubberBand::RubberBandStretcher stretcher;
     MixingPlayer *mPlayer;
     std::string mFilesDir;
+    std::unique_ptr<MP3File> musicFile;
+    std::atomic<bool> haveMusicFile;
+    std::unique_ptr<std::thread> musicFeed;
+    std::atomic<bool> exitMusicFeedThread;
+    int android_fd;
+    void initRubberBand();
+    void closeRubberBand();
+    void closeMusicFile();
+    void musicFeedThread();
 };
 
 #endif //LOCKSTEP_PLAYBACKENGINE_H

app/src/main/cpp/mp3file.cpp

@@ -151,7 +151,7 @@ MP3File* mp3file_open_fd(int fd, int forceEncoding) {
 
     mp3->offset = 0;
     mp3->remaining_samples = (int) mp3->num_samples;
-    LOGV("channels: %d rate: %ld num_samples: %ld", mp3->channels, mp3->rate, mp3->num_samples);
+    LOGI("channels: %d rate: %ld num_samples: %ld", mp3->channels, mp3->rate, mp3->num_samples);
     mp3->android_fd = fd;
     return mp3;
 

app/src/main/java/at/lockstep/app/LstForegroundService.java

@@ -87,7 +87,9 @@ public class LstForegroundService extends Service implements SensorEventListener
             if (ACTION_START.equals(action)) {
                 String contentUri = intent.getStringExtra("content_uri");
                 try {
-                    PlaybackEngine.playMusic(uriToFd(contentUri));
+                    if(contentUri != null) {
+                        PlaybackEngine.playMusic(uriToFd(contentUri));
+                    }
                 } catch (IOException e) {
                     // TODO proper error handling
                     Toast.makeText(this, "Could not open music file contentUri", Toast.LENGTH_LONG).show();