docs

TODO.md

@@ -3,6 +3,7 @@
 * analyze the (secondly or so) noise beeps in the mp3 playback
   - introduced with this commit
   - is it librubberband, my failure to feed it properly (buffer exhaustion), or sth else?
+  - the sizes of my buffers?
 
 * correct sampling rate of libmpg123 vs. 48000 Hz using librubberband
 

app/src/main/cpp/MixingPlayer.h

@@ -55,6 +55,9 @@ public:
         return (int32_t) result;
     }
 
+    int getRate() { return kSampleRate; }
+    int getNumChannels() { return kChannelCount; }
+
     // Call this from Activity onPause()
     void stopAudio() {
         // Stop, close and delete in case not already closed.

app/src/main/cpp/PlaybackEngine.cpp

@@ -54,7 +54,6 @@ static bool read_mp3(std::string filename, std::vector<float>& samples) {
     return ok1 && ok2;
 }
 
-
 struct RbLogger : public RubberBand::RubberBandStretcher::Logger {
     virtual void log(const char *s) {
         LOGI("%s", s);
@@ -80,24 +79,39 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid):
     mFilesDir(filesDir),
     haveMusicFile(false),
     exitMusicFeedThread(false),
-    android_fd(0)
+    android_fd(0),
+    haveTimeRatio(false),
+    timeRatio(1.0),
+
+    // these 3 values are preliminary -- will be set from MixingPlayer defaults in the ctor body below
+    playbackRate(48000),
+    numOutChannels(2),
+    numInChannels(2)
 {
     LOGI("PlaybackEngine()");
     LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL);
     // NDK LOG_LEVEL=3  (DEBUG)
+
+    // load "bump" sound effect
     std::vector<float> samples;
     bool is_ok = read_mp3(mFilesDir + "/" + std::to_string(resid) + ".mp3", samples);
-    LOGI("read_mp3() is_ok=%d", is_ok);
+    LOGI("read_mp3() for bump effect, is_ok=%d", is_ok);
+
     mPlayer = new MixingPlayer(samples);
     int32_t res = mPlayer->startAudio();
-    LOGI("startAudio() = %d", res);
+    playbackRate.store(mPlayer->getRate());
+    numOutChannels.store(mPlayer->getNumChannels());
+    LOGI("startAudio() = %d rate=%d channels=%d", res, playbackRate.load(), numOutChannels.load());
+
+    // configure stretcher and start musicFeedThread()
     initRubberBand();
 }
 
 void PlaybackEngine::initRubberBand() {
-    // TODO: check mp3 actual sample rate, and adapt to 48000 Hz playback here
+    // we do not yet have a music file with actual sampling rate, so set the default ratio
     stretcher.setTimeRatio(1.0);
     stretcher.setPitchScale(1.0);
+
     stretcher.setDebugLevel(1); // 1: errors only. generally 0..4
 
     // feed samples into 'stretcher' and read bogus output
@@ -147,39 +161,92 @@ void PlaybackEngine::closeMusicFile() {
     }
 }
 
+void PlaybackEngine::mapChannels(int *channel_map, int num_ch_in, int num_ch_out) {
+    if(num_ch_in == num_ch_out) {
+        // map each channel as-is
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = i;
+    } else if(num_ch_in == 1) {
+        // map mono to all output channels
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = 0;
+    } else if(num_ch_in >= 2) {
+        // use a stereo mapping
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = i;
+    } else {
+        LOGE("mapChannels(): strange channel layout, mapping to mono. num_ch_in=%d num_ch_out=%d", num_ch_in, num_ch_out);
+        // map mono to all output channels
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = 0;
+    }
+    // TODO: check broken input (0 channels etc) and bubble up an error to app
+}
+
 void PlaybackEngine::musicFeedThread() {
-    // refactor: rename to 'num_buf_samples'
+    // strecher num channels: same as output num channels
-    size_t num_pad = 48000; // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()?
+    // (this is because we play silence even without any input file, so we cannot set stretcher
-    size_t buf_stride = num_pad;
+    // channel count for the music file's channel count)
-    float* buf = (float*) malloc(num_pad*2*sizeof(float));
+    int num_ch_in = numInChannels.load();
-    float* buf_ptr[] {buf, buf + num_pad};
+    int num_ch_out = numOutChannels.load();
-    memset(buf, 0, num_pad*2*sizeof(float));
+
-    unsigned char* cbuf = (unsigned char*) malloc(num_pad*2*sizeof(int16_t));
+    size_t num_buf_samples = buf_size_samples;
-    memset(cbuf, 0, num_pad*2*sizeof(int16_t));
+    size_t buf_stride = num_buf_samples;
-    size_t cbuf_size_bytes = num_pad*2*sizeof(int16_t);
+    size_t buf_size_bytes = num_buf_samples * num_ch_out * sizeof(float);
+    float* buf = (float*) malloc(buf_size_bytes);
+    float** buf_ptr = (float**) malloc(num_ch_out * sizeof(float*));
+    for(int i = 0; i < num_ch_out; i++) {
+        buf_ptr[i] = buf + i * num_buf_samples;
+    }
+    memset(buf, 0, buf_size_bytes);
+    // preliminary allocation (actual music file buffer is unknown due to unknown channel count)
+    size_t cbuf_size_bytes = num_buf_samples * num_ch_in * sizeof(int16_t);
+    //size_t cbuf_load_bytes = num_buf_samples * num_ch_in * sizeof(int16_t);
+    unsigned char* cbuf = (unsigned char*) malloc(cbuf_size_bytes);
+    memset(cbuf, 0, cbuf_size_bytes);
+
+    int* channel_map = (int*) malloc(num_ch_out * sizeof(int));
+
+    int loop_delay_ms = 1000 * buf_size_samples / playbackRate.load();
 
     int idebug = 0;
 
     // thread 2: polling for decoding more mp3 -> process()  -- getSamplesRequired()
     while(!exitMusicFeedThread.load()) {
+
+        if(haveTimeRatio.load()) {
+            stretcher.setTimeRatio(timeRatio.load());
+            haveTimeRatio.store(false);
+        }
+
+        // change buffer size, if necessary (changed input channel count)
+        if(numInChannels.load() != num_ch_in) {
+            LOGD("changed buffer size (changed input channel count)");
+            num_ch_in = numInChannels.load();
+            free(cbuf);
+            cbuf_size_bytes = num_buf_samples * num_ch_in * sizeof(int16_t);
+            cbuf = (unsigned char*) malloc(cbuf_size_bytes);
+            memset(cbuf, 0, cbuf_size_bytes);
+        }
+
+        mapChannels(channel_map, num_ch_in, num_ch_out);
+
         // do work ...
         size_t num_samples = stretcher.getSamplesRequired();
 
         // note: how much to sleep until output has played x samples...?
-        // can we just measure the wall time here, instead of calculating? -- that will be imprecise.
         // how large is one buffer, and when do we feed it more data?
         // (is it like double-buffering implemented in 'stretcher'?)
 
-        // can we just wait some bogus interval here, for a first version?
         if (num_samples == 0) {
             //LOGD("waiting for getSamplesRequired()");
-            std::this_thread::sleep_for(std::chrono::milliseconds(20));
+            std::this_thread::sleep_for(std::chrono::milliseconds(loop_delay_ms));
             continue;
         }
 
-        if (num_samples > num_pad) {
+        if (num_samples > num_buf_samples) {
-            LOGE("wanted %d samples but buf is only %d samples", num_samples, num_pad);
+            LOGE("wanted %d samples but buf is only %d samples", num_samples, num_buf_samples);
-            continue;
+            num_samples = num_buf_samples;
         }
 
         if (!haveMusicFile.load()) {
@@ -188,7 +255,7 @@ void PlaybackEngine::musicFeedThread() {
                 // 1024, 512, 512
                 // 7 x 512
             }
-            memset(buf, 0, num_samples*2*sizeof(float));
+            memset(buf, 0, num_samples*num_ch_out*sizeof(float));
             stretcher.process(buf_ptr, num_samples, false);
             continue;
         }
@@ -203,7 +270,7 @@ void PlaybackEngine::musicFeedThread() {
         size_t done = 0; // bytes!
         int err = mpg123_read(musicFile->handle, cbuf, cbuf_size_bytes, &done);
         musicFile->remaining_samples -= done / sizeof(int16_t);
-        musicFile->offset = 0;
+        musicFile->offset = 0; // unused here
         if (err != MPG123_OK && err != MPG123_DONE) {
             // error!
             LOGE("mpg123_read() err=%d done=%d", err, done);
@@ -217,18 +284,25 @@ void PlaybackEngine::musicFeedThread() {
             continue;
         }
 
-        size_t num_decoded_samples = done / sizeof(int16_t) / 2; // 2 channels - TODO: actually use mp3 channels!! below, too. 2.
+        size_t num_decoded_samples = done / sizeof(int16_t) / num_ch_in;
-        LOGI("num_decoded_samples = %d", num_decoded_samples);
+        //LOGD("num_decoded_samples = %d", num_decoded_samples);
-        // convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
+
+        // * convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
+        // * map input to output channels
         for(size_t i = 0; i < num_decoded_samples; i++) {
-            for(size_t j = 0; j < 2; j++) {
+            for(size_t j = 0; j < num_ch_out; j++) {
-                buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i*2 + j)) / 32768.0f;
+                buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i * num_ch_in + channel_map[j])) / 32768.0f;
             }
         }
 
-        LOGI("calling stretcher.process()");
+        //LOGD("calling stretcher.process()");
         stretcher.process(buf_ptr, num_decoded_samples, false);
     }
+
+    free(buf);
+    free(buf_ptr);
+    free(cbuf);
+    free(channel_map);
 }
 
 PlaybackEngine::~PlaybackEngine() {
@@ -245,36 +319,34 @@ void PlaybackEngine::playBeat() {
 
 void PlaybackEngine::playMusic(int fd) {
     if(!mPlayer) return;
-    // TODO: fetch sampling rate from mp3 file, and use librubberband to correct for it
-    // MixingPlayer::kSampleRate (48000)
-    // mp3->rate
-
-    // feed samples to librubberband
-    // fetch resamples out of librubberband
-
-    //if(mPlayer) mPlayer->playMusic();
-    // TODO: fd is opened; dispose of fd when stopping or being discarded ...
     LOGI("PlaybackEngine::playMusic(fd=%d)", fd);
-    //close(fd); // for now, nothing is implemented. we just close it again.
-    // we will use mp3file_open_fd() later.
-
     android_fd = fd;
     musicFile.reset(mp3file_open_fd(android_fd, 0));
+    if(musicFile) {
+        timeRatio.store(((double) playbackRate.load()) / ((double) musicFile->rate));
+        haveTimeRatio.store(true);
+        numInChannels.store(musicFile->channels);
         haveMusicFile.store(true);
-    mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher));
+    }
+    mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher, buf_size_samples, numOutChannels.load()));
 }
 
-MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher) : stretcher(stretcher), idebug(0) {
+MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples, int num_ch_out) :
-    // refactor: rename to 'num_buf_samples'
+    stretcher(stretcher),
-    // TODO: for cache-friendliness, it would be better to have smaller 'num_buf_samples'
+    idebug(0),
-    // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()?
+    buf_size_samples(buf_size_samples),
-    //size_t buf_stride = num_pad;
+    num_ch_out(num_ch_out)
-    buf = (float*) malloc(num_buf_samples*2*sizeof(float));
+{
-    //float* buf_ptr[] {buf, buf + num_pad};
+    buf = (float*) malloc(buf_size_samples*num_ch_out*sizeof(float));
+    buf_ptr = (float**) malloc(num_ch_out * sizeof(float*));
+    for(int i = 0; i < num_ch_out; i++) {
+        buf_ptr[i] = buf + i * buf_size_samples;
+    }
 }
 
 MusicProvider::~MusicProvider() {
     free(buf);
+    free(buf_ptr);
 }
 
 void MusicProvider::onAudioReady(float *data, int32_t frames) {
@@ -283,18 +355,21 @@ void MusicProvider::onAudioReady(float *data, int32_t frames) {
         // frames=96 (48 kHz => 2 ms!!)
     }
 
+    if(frames > buf_size_samples) {
+        LOGE("MusicProvider::onAudioReady() asked for frames=%d but buf_size=%d", frames, buf_size_samples);
+    }
+
     // 1. read from oboe into our temp de-interleaved buffer 'buf'
-    size_t num_frames = std::min((size_t) frames, num_buf_samples);
+    size_t num_frames = std::min((size_t) frames, buf_size_samples);
-    float* buf_ptr[] {buf, buf + num_buf_samples};
     stretcher->retrieve(buf_ptr, num_frames);
 
     // 2. convert to add samples to interleaved *data
     for(size_t i = 0; i < num_frames; i++) {
-        for(size_t j = 0; j < 2; j++) {
+        for(size_t j = 0; j < num_ch_out; j++) {
-            float sample = data[i*2 + j];
+            float sample = data[i*num_ch_out + j];
             sample += buf_ptr[j][i];
             sample /= 2.0;
-            data[i*2 + j] = sample;
+            data[i*num_ch_out + j] = sample;
         }
     }
 }

app/src/main/cpp/PlaybackEngine.h

@@ -18,16 +18,18 @@
 /** Provides music through a regular callback to oboe. Called from separate oboe thread. */
 class MusicProvider : public AudioCallbackProvider {
 public:
-    explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher);
+    explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples, int num_ch_out);
     ~MusicProvider() override;
 
     /** Called from separate oboe thread. */
     void onAudioReady(float *data, int32_t frames) override;
 private:
-    const size_t num_buf_samples = 48000;
     RubberBand::RubberBandStretcher *stretcher;
     float *buf;
+    float **buf_ptr;
     int idebug;
+    size_t buf_size_samples;
+    int num_ch_out;
 };
 
 class PlaybackEngine : public StepListener {
@@ -46,10 +48,18 @@ private:
     std::unique_ptr<std::thread> musicFeed;
     std::atomic<bool> exitMusicFeedThread;
     int android_fd;
+    std::atomic<bool> haveTimeRatio;
+    std::atomic<double> timeRatio;
+    std::atomic<int> playbackRate;
+    std::atomic<int> numOutChannels;
+    std::atomic<int> numInChannels;
+    /** this is actually in frames, not samples */
+    static size_t constexpr buf_size_samples = 1024;
     void initRubberBand();
     void closeRubberBand();
     void closeMusicFile();
     void musicFeedThread();
+    void mapChannels(int *channel_map, int num_ch_in, int num_ch_out);
 };
 
 #endif //LOCKSTEP_PLAYBACKENGINE_H

app/src/main/cpp/mp3file.h

@@ -13,6 +13,7 @@ struct MP3File
     int android_fd;
     int channels;
     long rate;
+    /** num samples in total (stereo of 10 frames will have 20 'samples' here) */
     long num_samples;
     int samples_per_frame;
     double secs_per_frame;
@@ -20,6 +21,7 @@ struct MP3File
     double duration;
     size_t buffer_size;
     unsigned char* buffer;
+    /** total samples (stereo of 10 frames remaining will have 20 'remaining_samples' here) */
     int remaining_samples;
     size_t offset;
 };