feat: actually map channels

app/src/main/cpp/MixingPlayer.h

@@ -56,6 +56,7 @@ public:
     }
 
     int getRate() { return kSampleRate; }
+    int getNumChannels() { return kChannelCount; }
 
     // Call this from Activity onPause()
     void stopAudio() {

app/src/main/cpp/PlaybackEngine.cpp

@@ -83,7 +83,9 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid):
     android_fd(0),
     haveTimeRatio(false),
     timeRatio(1.0),
-    playbackRate(48000)
+    playbackRate(48000),
+    numOutChannels(2),
+    numInChannels(2)
 {
     LOGI("PlaybackEngine()");
     LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL);
@@ -93,8 +95,9 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid):
     LOGI("read_mp3() is_ok=%d", is_ok);
     mPlayer = new MixingPlayer(samples);
     int32_t res = mPlayer->startAudio();
-    playbackRate = mPlayer->getRate();
-    LOGI("startAudio() = %d", res);
+    playbackRate.store(mPlayer->getRate());
+    numOutChannels.store(mPlayer->getNumChannels());
+    LOGI("startAudio() = %d rate=%d channels=%d", res, playbackRate.load(), numOutChannels.load());
     initRubberBand();
 }
 
@@ -151,15 +154,53 @@ void PlaybackEngine::closeMusicFile() {
     }
 }
 
+void PlaybackEngine::mapChannels(int *channel_map, int num_ch_in, int num_ch_out) {
+    if(num_ch_in == num_ch_out) {
+        // map each channel as-is
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = i;
+    } else if(num_ch_in == 1) {
+        // map mono to all output channels
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = 0;
+    } else if(num_ch_in >= 2) {
+        // use a stereo mapping
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = i;
+    } else {
+        LOGE("mapChannels(): strange channel layout, mapping to mono. num_ch_in=%d num_ch_out=%d", num_ch_in, num_ch_out);
+        // map mono to all output channels
+        for(int i = 0; i < num_ch_out; i++)
+            channel_map[i] = 0;
+    }
+    // TODO: check broken input (0 channels etc) and bubble up an error to app
+}
+
 void PlaybackEngine::musicFeedThread() {
+    // strecher num channels: same as output num channels
+    // (this is because we play silence even without any input file, so we cannot set stretcher
+    // channel count for the music file's channel count)
+    int num_ch_in = numInChannels.load();
+    int num_ch_out = numOutChannels.load();
+
     size_t num_buf_samples = buf_size_samples;
     size_t buf_stride = num_buf_samples;
-    float* buf = (float*) malloc(num_buf_samples * 2 * sizeof(float));
-    float* buf_ptr[] {buf, buf + num_buf_samples};
-    memset(buf, 0, num_buf_samples * 2 * sizeof(float));
-    unsigned char* cbuf = (unsigned char*) malloc(num_buf_samples * 2 * sizeof(int16_t));
-    memset(cbuf, 0, num_buf_samples * 2 * sizeof(int16_t));
-    size_t cbuf_size_bytes = num_buf_samples * 2 * sizeof(int16_t);
+    size_t buf_size_bytes = num_buf_samples * num_ch_out * sizeof(float);
+    float* buf = (float*) malloc(buf_size_bytes);
+    float** buf_ptr = (float**) malloc(num_ch_out * sizeof(float*));
+    for(int i = 0; i < num_ch_out; i++) {
+        buf_ptr[i] = buf + i * num_buf_samples;
+    }
+    memset(buf, 0, buf_size_bytes);
+    // preliminary allocation (actual music file buffer is unknown due to unknown channel count)
+    size_t cbuf_size_bytes = num_buf_samples * num_ch_in * sizeof(int16_t);
+    //size_t cbuf_load_bytes = num_buf_samples * num_ch_in * sizeof(int16_t);
+    unsigned char* cbuf = (unsigned char*) malloc(cbuf_size_bytes);
+    memset(cbuf, 0, cbuf_size_bytes);
+
+    int* channel_map = (int*) malloc(num_ch_out * sizeof(int));
+
+    int loop_delay_ms = 1000 * buf_size_samples / playbackRate.load();
 
     int idebug = 0;
 
@@ -167,28 +208,38 @@ void PlaybackEngine::musicFeedThread() {
     while(!exitMusicFeedThread.load()) {
 
         if(haveTimeRatio.load()) {
-            stretcher.setTimeRatio(timeRatio);
+            stretcher.setTimeRatio(timeRatio.load());
             haveTimeRatio.store(false);
         }
 
+        // change buffer size, if necessary (changed input channel count)
+        if(numInChannels.load() != num_ch_in) {
+            LOGD("changed buffer size (changed input channel count)");
+            num_ch_in = numInChannels.load();
+            free(cbuf);
+            cbuf_size_bytes = num_buf_samples * num_ch_in * sizeof(int16_t);
+            cbuf = (unsigned char*) malloc(cbuf_size_bytes);
+            memset(cbuf, 0, cbuf_size_bytes);
+        }
+
+        mapChannels(channel_map, num_ch_in, num_ch_out);
+
         // do work ...
         size_t num_samples = stretcher.getSamplesRequired();
 
         // note: how much to sleep until output has played x samples...?
-        // can we just measure the wall time here, instead of calculating? -- that will be imprecise.
         // how large is one buffer, and when do we feed it more data?
         // (is it like double-buffering implemented in 'stretcher'?)
 
-        // can we just wait some bogus interval here, for a first version?
         if (num_samples == 0) {
             //LOGD("waiting for getSamplesRequired()");
-            std::this_thread::sleep_for(std::chrono::milliseconds(20));
+            std::this_thread::sleep_for(std::chrono::milliseconds(loop_delay_ms));
             continue;
         }
 
         if (num_samples > num_buf_samples) {
             LOGE("wanted %d samples but buf is only %d samples", num_samples, num_buf_samples);
-            continue;
+            num_samples = num_buf_samples;
         }
 
         if (!haveMusicFile.load()) {
@@ -197,7 +248,7 @@ void PlaybackEngine::musicFeedThread() {
                 // 1024, 512, 512
                 // 7 x 512
             }
-            memset(buf, 0, num_samples*2*sizeof(float));
+            memset(buf, 0, num_samples*num_ch_out*sizeof(float));
             stretcher.process(buf_ptr, num_samples, false);
             continue;
         }
@@ -212,7 +263,7 @@ void PlaybackEngine::musicFeedThread() {
         size_t done = 0; // bytes!
         int err = mpg123_read(musicFile->handle, cbuf, cbuf_size_bytes, &done);
         musicFile->remaining_samples -= done / sizeof(int16_t);
-        musicFile->offset = 0;
+        musicFile->offset = 0; // unused here
         if (err != MPG123_OK && err != MPG123_DONE) {
             // error!
             LOGE("mpg123_read() err=%d done=%d", err, done);
@@ -226,19 +277,25 @@ void PlaybackEngine::musicFeedThread() {
             continue;
         }
 
-        size_t num_decoded_samples = done / sizeof(int16_t) / 2; // 2 channels - TODO: actually use mp3 channels!! below, too. 2.
+        size_t num_decoded_samples = done / sizeof(int16_t) / num_ch_in;
         //LOGD("num_decoded_samples = %d", num_decoded_samples);
 
-        // convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
+        // * convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
+        // * map input to output channels
         for(size_t i = 0; i < num_decoded_samples; i++) {
-            for(size_t j = 0; j < 2; j++) {
-                buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i*2 + j)) / 32768.0f;
+            for(size_t j = 0; j < num_ch_out; j++) {
+                buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i * num_ch_in + channel_map[j])) / 32768.0f;
             }
         }
 
         //LOGD("calling stretcher.process()");
         stretcher.process(buf_ptr, num_decoded_samples, false);
     }
+
+    free(buf);
+    free(buf_ptr);
+    free(cbuf);
+    free(channel_map);
 }
 
 PlaybackEngine::~PlaybackEngine() {
@@ -259,24 +316,30 @@ void PlaybackEngine::playMusic(int fd) {
     android_fd = fd;
     musicFile.reset(mp3file_open_fd(android_fd, 0));
     if(musicFile) {
-        timeRatio = ((double) playbackRate) / ((double) musicFile->rate);
+        timeRatio.store(((double) playbackRate.load()) / ((double) musicFile->rate));
         haveTimeRatio.store(true);
-    }
+        numInChannels.store(musicFile->channels);
         haveMusicFile.store(true);
-    mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher, buf_size_samples));
+    }
+    mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher, buf_size_samples, numOutChannels.load()));
 }
 
-MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples) :
+MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples, int num_ch_out) :
     stretcher(stretcher),
     idebug(0),
-    buf_size_samples(buf_size_samples)
+    buf_size_samples(buf_size_samples),
+    num_ch_out(num_ch_out)
 {
-    buf = (float*) malloc(buf_size_samples*2*sizeof(float));
-    //float* buf_ptr[] {buf, buf + num_pad};
+    buf = (float*) malloc(buf_size_samples*num_ch_out*sizeof(float));
+    buf_ptr = (float**) malloc(num_ch_out * sizeof(float*));
+    for(int i = 0; i < num_ch_out; i++) {
+        buf_ptr[i] = buf + i * buf_size_samples;
+    }
 }
 
 MusicProvider::~MusicProvider() {
     free(buf);
+    free(buf_ptr);
 }
 
 void MusicProvider::onAudioReady(float *data, int32_t frames) {
@@ -285,18 +348,21 @@ void MusicProvider::onAudioReady(float *data, int32_t frames) {
         // frames=96 (48 kHz => 2 ms!!)
     }
 
+    if(frames > buf_size_samples) {
+        LOGE("MusicProvider::onAudioReady() asked for frames=%d but buf_size=%d", frames, buf_size_samples);
+    }
+
     // 1. read from oboe into our temp de-interleaved buffer 'buf'
     size_t num_frames = std::min((size_t) frames, buf_size_samples);
-    float* buf_ptr[] {buf, buf + buf_size_samples};
     stretcher->retrieve(buf_ptr, num_frames);
 
     // 2. convert to add samples to interleaved *data
     for(size_t i = 0; i < num_frames; i++) {
-        for(size_t j = 0; j < 2; j++) {
-            float sample = data[i*2 + j];
+        for(size_t j = 0; j < num_ch_out; j++) {
+            float sample = data[i*num_ch_out + j];
             sample += buf_ptr[j][i];
             sample /= 2.0;
-            data[i*2 + j] = sample;
+            data[i*num_ch_out + j] = sample;
         }
     }
 }

app/src/main/cpp/PlaybackEngine.h

@@ -18,7 +18,7 @@
 /** Provides music through a regular callback to oboe. Called from separate oboe thread. */
 class MusicProvider : public AudioCallbackProvider {
 public:
-    explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples);
+    explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher, size_t buf_size_samples, int num_ch_out);
     ~MusicProvider() override;
 
     /** Called from separate oboe thread. */
@@ -26,8 +26,10 @@ public:
 private:
     RubberBand::RubberBandStretcher *stretcher;
     float *buf;
+    float **buf_ptr;
     int idebug;
     size_t buf_size_samples;
+    int num_ch_out;
 };
 
 class PlaybackEngine : public StepListener {
@@ -47,13 +49,17 @@ private:
     std::atomic<bool> exitMusicFeedThread;
     int android_fd;
     std::atomic<bool> haveTimeRatio;
-    double timeRatio;
-    int playbackRate;
+    std::atomic<double> timeRatio;
+    std::atomic<int> playbackRate;
+    std::atomic<int> numOutChannels;
+    std::atomic<int> numInChannels;
+    /** this is actually in frames, not samples */
     static size_t constexpr buf_size_samples = 1024;
     void initRubberBand();
     void closeRubberBand();
     void closeMusicFile();
     void musicFeedThread();
+    void mapChannels(int *channel_map, int num_ch_in, int num_ch_out);
 };
 
 #endif //LOCKSTEP_PLAYBACKENGINE_H

app/src/main/cpp/mp3file.h

@@ -13,6 +13,7 @@ struct MP3File
     int android_fd;
     int channels;
     long rate;
+    /** num samples in total (stereo of 10 frames will have 20 'samples' here) */
     long num_samples;
     int samples_per_frame;
     double secs_per_frame;
@@ -20,6 +21,7 @@ struct MP3File
     double duration;
     size_t buffer_size;
     unsigned char* buffer;
+    /** total samples (stereo of 10 frames remaining will have 20 'remaining_samples' here) */
     int remaining_samples;
     size_t offset;
 };