feat: MixingPlayer - plays event sounds

TODO.md

@@ -0,0 +1,10 @@
+* myband PlaybackEngine.cpp has latency management and other audio performance related features.
+  Check if the app can be improved (audio wise) by using that code instead.
+
+* Sampling rate for accelerometer - do we need to measure actual sensor FPS, or is it stable 50 Hz?
+
+* re-calculate IIR filter coefficients. probably not critical for 50 Hz vs. 60 Hz.
+
+* re-visit sampling rate and channel count.
+  MixingPlayer currently forces both to 48000 and 2 respectively,
+  regardless of what Android says would be optimal.

app/src/main/cpp/MixingPlayer.h

@@ -0,0 +1,117 @@
+//
+// Created by david on 01.02.2026.
+//
+
+#ifndef LOCKSTEP_MIXINGPLAYER_H
+#define LOCKSTEP_MIXINGPLAYER_H
+
+#include <oboe/Oboe.h>
+#include <math.h>
+using namespace oboe;
+
+/**
+ * Plays event sounds, potentially overlapping in time.
+ * Use <c>setStartBeat()</c> to trigger playing sound for an event.
+ */
+class MixingPlayer: public oboe::AudioStreamDataCallback {
+protected:
+    std::vector<float> beatSound;
+    std::vector<int> beatIdx;
+    std::atomic<int> startBeat;
+    int numBeatsPlaying;
+public:
+    explicit MixingPlayer(std::vector<float> beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0) {}
+
+    virtual ~MixingPlayer() = default;
+
+    /**
+     * Play a beat sound by setting the startBeat register.
+     * A delay of the audio callback interval will be added.
+     */
+    void setStartBeat() {
+        startBeat.store(1);
+    }
+
+    // Call this from Activity onResume()
+    int32_t startAudio() {
+        std::lock_guard<std::mutex> lock(mLock);
+        oboe::AudioStreamBuilder builder;
+        // The builder set methods can be chained for convenience.
+        Result result = builder.setSharingMode(oboe::SharingMode::Exclusive)
+                ->setPerformanceMode(oboe::PerformanceMode::LowLatency)
+                ->setChannelCount(kChannelCount)
+                ->setSampleRate(kSampleRate)
+                ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium)
+                ->setFormat(oboe::AudioFormat::Float)
+                ->setDataCallback(this)
+                ->openStream(mStream);
+        if (result != Result::OK) return (int32_t) result;
+
+        // Typically, start the stream after querying some stream information, as well as some input from the user
+        result = mStream->requestStart();
+        return (int32_t) result;
+    }
+
+    // Call this from Activity onPause()
+    void stopAudio() {
+        // Stop, close and delete in case not already closed.
+        std::lock_guard<std::mutex> lock(mLock);
+        if (mStream) {
+            mStream->stop();
+            mStream->close();
+            mStream.reset();
+        }
+    }
+
+    oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override {
+        // fetch startBeat register
+        int isStartBeat = startBeat.load();
+        if(isStartBeat) {
+            if(beatIdx.size() < numBeatsPlaying+1) beatIdx.resize(numBeatsPlaying+1);
+            beatIdx[numBeatsPlaying++] = 0;
+            startBeat.store(0); // reset startBeat register
+        }
+        //
+        // mix audio events, each event has a separate idx counter in beatIdx
+        //
+        float *floatData = (float *) audioData;
+        int N = (int) beatSound.size();
+        for (int i = 0; i < numFrames; i++) {
+            float sample = 0.0;
+            float norm = (float) numBeatsPlaying;
+            for (int k = 0; k < numBeatsPlaying; k++) {
+                sample += beatSound[beatIdx[k]];
+                beatIdx[k] += 1;
+            }
+            // reduce beatIdx to the events which have not completed playing
+            int l = 0;
+            int m = 0;
+            for (; m < numBeatsPlaying; l++, m++) {
+                while(m < numBeatsPlaying && beatIdx[m] >= N) m++;
+                if(m < numBeatsPlaying)
+                    beatIdx[l] = beatIdx[m];
+                else
+                    break; // avoid incrementing l
+            }
+            //beatIdx.resize(l); // we avoid re-allocating
+            numBeatsPlaying = l;
+            // set left and right output channels
+            for (int j = 0; j < kChannelCount; j++) {
+                // normalize sample by numBeatsPlaying (avoids clipping)
+                floatData[i * kChannelCount + j] = sample / norm;
+            }
+        }
+
+        return oboe::DataCallbackResult::Continue;
+    }
+
+private:
+    std::mutex         mLock;
+    std::shared_ptr<oboe::AudioStream> mStream;
+
+    // Stream params
+    static int constexpr kChannelCount = 2;
+    static int constexpr kSampleRate = 48000;
+};
+
+#endif //LOCKSTEP_MIXINGPLAYER_H

app/src/main/cpp/OboeSinePlayer.h

@@ -1,78 +0,0 @@
-//
-// Created by david on 01.02.2026.
-//
-
-#ifndef LOCKSTEP_OBOESINEPLAYER_H
-#define LOCKSTEP_OBOESINEPLAYER_H
-
-#include <oboe/Oboe.h>
-#include <math.h>
-using namespace oboe;
-
-class OboeSinePlayer: public oboe::AudioStreamDataCallback {
-public:
-
-    virtual ~OboeSinePlayer() = default;
-
-    // Call this from Activity onResume()
-    int32_t startAudio() {
-        std::lock_guard<std::mutex> lock(mLock);
-        oboe::AudioStreamBuilder builder;
-        // The builder set methods can be chained for convenience.
-        Result result = builder.setSharingMode(oboe::SharingMode::Exclusive)
-                ->setPerformanceMode(oboe::PerformanceMode::LowLatency)
-                ->setChannelCount(kChannelCount)
-                ->setSampleRate(kSampleRate)
-                ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium)
-                ->setFormat(oboe::AudioFormat::Float)
-                ->setDataCallback(this)
-                ->openStream(mStream);
-        if (result != Result::OK) return (int32_t) result;
-
-        // Typically, start the stream after querying some stream information, as well as some input from the user
-        result = mStream->requestStart();
-        return (int32_t) result;
-    }
-
-    // Call this from Activity onPause()
-    void stopAudio() {
-        // Stop, close and delete in case not already closed.
-        std::lock_guard<std::mutex> lock(mLock);
-        if (mStream) {
-            mStream->stop();
-            mStream->close();
-            mStream.reset();
-        }
-    }
-
-    oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override {
-        float *floatData = (float *) audioData;
-        for (int i = 0; i < numFrames; ++i) {
-            float sampleValue = kAmplitude * sinf(mPhase);
-            for (int j = 0; j < kChannelCount; j++) {
-                floatData[i * kChannelCount + j] = sampleValue;
-            }
-            mPhase += mPhaseIncrement;
-            if (mPhase >= kTwoPi) mPhase -= kTwoPi;
-        }
-        return oboe::DataCallbackResult::Continue;
-    }
-
-private:
-    std::mutex         mLock;
-    std::shared_ptr<oboe::AudioStream> mStream;
-
-    // Stream params
-    static int constexpr kChannelCount = 2;
-    static int constexpr kSampleRate = 48000;
-    // Wave params, these could be instance variables in order to modify at runtime
-    static float constexpr kAmplitude = 0.5f;
-    static float constexpr kFrequency = 440;
-    static float constexpr kPI = M_PI;
-    static float constexpr kTwoPi = kPI * 2;
-    static double constexpr mPhaseIncrement = kFrequency * kTwoPi / (double) kSampleRate;
-    // Keeps track of where the wave is
-    float mPhase = 0.0;
-};
-
-#endif //LOCKSTEP_OBOESINEPLAYER_H

app/src/main/cpp/PlaybackEngine.cpp

@@ -6,12 +6,46 @@
 
 #include "PlaybackEngine.h"
 #include "logging.h"
+#include "mpg123.h"
+#include "mp3file.h"
+#include <vector>
 
-PlaybackEngine::PlaybackEngine(std::string filesDir): mFilesDir(filesDir) {
+static inline int readBuffer2(MP3File* mp3)
+{
+    size_t done = 0;
+    int err = mpg123_read(mp3->handle, mp3->buffer, mp3->buffer_size, &done);
+    mp3->leftSamples = done / sizeof(int16_t);
+    mp3->offset = 0;
+    return err != MPG123_OK ? 0 : done;
+}
+
+static bool read_mp3(std::string filename, std::vector<float>& samples) {
+    // TODO: assumes 48000 Hz sampling rate of the file
+    // note: our resource file is mono (1 channel = 1 samples_per_frame)!
+
+    MP3File *mFile = mp3file_open(filename.c_str(), 0); // MPG123_ENC_FLOAT_32 (but does not seem to work)
+    bool ok1 = mFile != nullptr;
+    bool ok2 = readBuffer2(mFile) != 0; // once is enough (maybe the other one needs to pre-fill other buffers, in oboe etc.)
+    if(ok1 && ok2) {
+        // num_frames, num_samples, samples_per_frame
+        samples.resize(mFile->num_samples);
+        for(int i = 0; i < mFile->num_samples; i++) {
+            int16_t *src = ((int16_t *) mFile->buffer) + i;
+            samples[i] = (*src) / 32768.0f;
+        }
+    }
+    if(mFile)
+        mp3file_delete(mFile);
+    return ok1 && ok2;
+}
+
+PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(filesDir) {
     LOGI("PlaybackEngine()");
     LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL);
     // NDK LOG_LEVEL=3  (DEBUG)
-    mPlayer = new OboeSinePlayer();
+    std::vector<float> samples;
+    read_mp3(mFilesDir + "/" + std::to_string(resid) + ".mp3", samples);
+    mPlayer = new MixingPlayer(samples);
     int32_t res = mPlayer->startAudio();
     LOGI("startAudio() = %d", res);
 }

app/src/main/cpp/PlaybackEngine.h

@@ -5,15 +5,15 @@
 #ifndef LOCKSTEP_PLAYBACKENGINE_H
 #define LOCKSTEP_PLAYBACKENGINE_H
 
-#include "OboeSinePlayer.h"
+#include "MixingPlayer.h"
 #include <string>
 
 class PlaybackEngine {
 public:
-    PlaybackEngine(std::string filesDir);
+    PlaybackEngine(std::string filesDir, int resid);
     virtual ~PlaybackEngine();
 private:
-    OboeSinePlayer *mPlayer;
+    MixingPlayer *mPlayer;
     std::string mFilesDir;
 };
 

app/src/main/cpp/lockstep.cpp

@@ -30,13 +30,13 @@ extern "C" {
 JNIEXPORT jlong JNICALL
 Java_at_lockstep_pb_PlaybackEngine_native_1createEngine(
         JNIEnv *env,
-        jclass /*unused*/, jstring filesDir) {
+        jclass /*unused*/, jstring filesDir, jint resid) {
     const char* filesDirTemp = env->GetStringUTFChars(filesDir, NULL);
     std::string filesDirString(filesDirTemp);
     env->ReleaseStringUTFChars(filesDir, filesDirTemp);
 
     // We use std::nothrow so `new` returns a nullptr if the engine creation fails
-    auto *engine = new(std::nothrow) PlaybackEngine(filesDirString);
+    auto *engine = new(std::nothrow) PlaybackEngine(filesDirString, resid);
     return reinterpret_cast<jlong>(engine);
 }
 

app/src/main/java/at/lockstep/app/MainActivity.java

@@ -12,7 +12,8 @@ public class MainActivity extends Activity {
     @Override
     protected void onResume() {
         super.onResume();
-        PlaybackEngine.create(this); // note: called twice (is permission request causing Activity to go out of focus?)
+        int resid = at.lockstep.R.raw.track_beat;
+        PlaybackEngine.create(this, resid); // note: called twice (is permission request causing Activity to go out of focus?)
     }
 
     @Override

app/src/main/java/at/lockstep/pb/PlaybackEngine.java

@@ -14,12 +14,19 @@ public class PlaybackEngine {
 
     static {
         System.loadLibrary("lockstep-native");
+        // mpg123_init() must be called before createEngine()
         int ok = native_mpg123_init();
         if(ok != MPG123_OK)
             throw new IllegalStateException("mpg123_init() failed");
     }
 
-    public static boolean create(Context context) {
+    /**
+     * Initialize Oboe library and start playing silence.
+     * @param context Context for filesystem operations (e.g. Activity)
+     * @param resid Beat sound (step detection audio-feedback sample)
+     * @return true if successful
+     */
+    public static boolean create(Context context, int resid) {
         try {
             if(!mFilesystemInitialized) {
                 AudioResources.copyRawTracksToFilesystem(context);
@@ -33,12 +40,16 @@ public class PlaybackEngine {
         if (mEngineHandle == 0) {
             setDefaultStreamValues(context);
             Log.i("PlaybackEngine", "Hello PlaybackEngine");
-            mEngineHandle = native_createEngine(context.getFilesDir().toString());
+            mEngineHandle = native_createEngine(context.getFilesDir().toString(), resid);
         }
         return (mEngineHandle != 0);
     }
 
     private static void setDefaultStreamValues(Context context) {
+        // nice-to:
+        // * re-visit sampling rate and channel count.
+        //   In C++ code, MixingPlayer currently forces both to 48000 and 2 respectively,
+        //   regardless of what Android says would be optimal.
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1){
             AudioManager myAudioMgr = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE);
             String sampleRateStr = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE);
@@ -57,7 +68,7 @@ public class PlaybackEngine {
         mEngineHandle = 0;
     }
 
-    private static native long native_createEngine(String filesDir);
+    private static native long native_createEngine(String filesDir, int resid);
     private static native void native_deleteEngine(long engineHandle);
     private static native void native_setDefaultStreamValues(int sampleRate, int framesPerBurst);