diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..0fa7706 --- /dev/null +++ b/TODO.md @@ -0,0 +1,10 @@ +* myband PlaybackEngine.cpp has latency management and other audio performance related features. + Check if the app can be improved (audio wise) by using that code instead. + +* Sampling rate for accelerometer - do we need to measure actual sensor FPS, or is it stable 50 Hz? + +* re-calculate IIR filter coefficients. probably not critical for 50 Hz vs. 60 Hz. + +* re-visit sampling rate and channel count. + MixingPlayer currently forces both to 48000 and 2 respectively, + regardless of what Android says would be optimal. diff --git a/app/src/main/cpp/MixingPlayer.h b/app/src/main/cpp/MixingPlayer.h new file mode 100644 index 0000000..a1893ef --- /dev/null +++ b/app/src/main/cpp/MixingPlayer.h @@ -0,0 +1,117 @@ +// +// Created by david on 01.02.2026. +// + +#ifndef LOCKSTEP_MIXINGPLAYER_H +#define LOCKSTEP_MIXINGPLAYER_H + +#include +#include +using namespace oboe; + +/** + * Plays event sounds, potentially overlapping in time. + * Use setStartBeat() to trigger playing sound for an event. + */ +class MixingPlayer: public oboe::AudioStreamDataCallback { +protected: + std::vector beatSound; + std::vector beatIdx; + std::atomic startBeat; + int numBeatsPlaying; +public: + explicit MixingPlayer(std::vector beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0) {} + + virtual ~MixingPlayer() = default; + + /** + * Play a beat sound by setting the startBeat register. + * A delay of the audio callback interval will be added. + */ + void setStartBeat() { + startBeat.store(1); + } + + // Call this from Activity onResume() + int32_t startAudio() { + std::lock_guard lock(mLock); + oboe::AudioStreamBuilder builder; + // The builder set methods can be chained for convenience. + Result result = builder.setSharingMode(oboe::SharingMode::Exclusive) + ->setPerformanceMode(oboe::PerformanceMode::LowLatency) + ->setChannelCount(kChannelCount) + ->setSampleRate(kSampleRate) + ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium) + ->setFormat(oboe::AudioFormat::Float) + ->setDataCallback(this) + ->openStream(mStream); + if (result != Result::OK) return (int32_t) result; + + // Typically, start the stream after querying some stream information, as well as some input from the user + result = mStream->requestStart(); + return (int32_t) result; + } + + // Call this from Activity onPause() + void stopAudio() { + // Stop, close and delete in case not already closed. + std::lock_guard lock(mLock); + if (mStream) { + mStream->stop(); + mStream->close(); + mStream.reset(); + } + } + + oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override { + // fetch startBeat register + int isStartBeat = startBeat.load(); + if(isStartBeat) { + if(beatIdx.size() < numBeatsPlaying+1) beatIdx.resize(numBeatsPlaying+1); + beatIdx[numBeatsPlaying++] = 0; + startBeat.store(0); // reset startBeat register + } + // + // mix audio events, each event has a separate idx counter in beatIdx + // + float *floatData = (float *) audioData; + int N = (int) beatSound.size(); + for (int i = 0; i < numFrames; i++) { + float sample = 0.0; + float norm = (float) numBeatsPlaying; + for (int k = 0; k < numBeatsPlaying; k++) { + sample += beatSound[beatIdx[k]]; + beatIdx[k] += 1; + } + // reduce beatIdx to the events which have not completed playing + int l = 0; + int m = 0; + for (; m < numBeatsPlaying; l++, m++) { + while(m < numBeatsPlaying && beatIdx[m] >= N) m++; + if(m < numBeatsPlaying) + beatIdx[l] = beatIdx[m]; + else + break; // avoid incrementing l + } + //beatIdx.resize(l); // we avoid re-allocating + numBeatsPlaying = l; + // set left and right output channels + for (int j = 0; j < kChannelCount; j++) { + // normalize sample by numBeatsPlaying (avoids clipping) + floatData[i * kChannelCount + j] = sample / norm; + } + } + + return oboe::DataCallbackResult::Continue; + } + +private: + std::mutex mLock; + std::shared_ptr mStream; + + // Stream params + static int constexpr kChannelCount = 2; + static int constexpr kSampleRate = 48000; +}; + +#endif //LOCKSTEP_MIXINGPLAYER_H diff --git a/app/src/main/cpp/OboeSinePlayer.h b/app/src/main/cpp/OboeSinePlayer.h deleted file mode 100644 index 95863e8..0000000 --- a/app/src/main/cpp/OboeSinePlayer.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by david on 01.02.2026. -// - -#ifndef LOCKSTEP_OBOESINEPLAYER_H -#define LOCKSTEP_OBOESINEPLAYER_H - -#include -#include -using namespace oboe; - -class OboeSinePlayer: public oboe::AudioStreamDataCallback { -public: - - virtual ~OboeSinePlayer() = default; - - // Call this from Activity onResume() - int32_t startAudio() { - std::lock_guard lock(mLock); - oboe::AudioStreamBuilder builder; - // The builder set methods can be chained for convenience. - Result result = builder.setSharingMode(oboe::SharingMode::Exclusive) - ->setPerformanceMode(oboe::PerformanceMode::LowLatency) - ->setChannelCount(kChannelCount) - ->setSampleRate(kSampleRate) - ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium) - ->setFormat(oboe::AudioFormat::Float) - ->setDataCallback(this) - ->openStream(mStream); - if (result != Result::OK) return (int32_t) result; - - // Typically, start the stream after querying some stream information, as well as some input from the user - result = mStream->requestStart(); - return (int32_t) result; - } - - // Call this from Activity onPause() - void stopAudio() { - // Stop, close and delete in case not already closed. - std::lock_guard lock(mLock); - if (mStream) { - mStream->stop(); - mStream->close(); - mStream.reset(); - } - } - - oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override { - float *floatData = (float *) audioData; - for (int i = 0; i < numFrames; ++i) { - float sampleValue = kAmplitude * sinf(mPhase); - for (int j = 0; j < kChannelCount; j++) { - floatData[i * kChannelCount + j] = sampleValue; - } - mPhase += mPhaseIncrement; - if (mPhase >= kTwoPi) mPhase -= kTwoPi; - } - return oboe::DataCallbackResult::Continue; - } - -private: - std::mutex mLock; - std::shared_ptr mStream; - - // Stream params - static int constexpr kChannelCount = 2; - static int constexpr kSampleRate = 48000; - // Wave params, these could be instance variables in order to modify at runtime - static float constexpr kAmplitude = 0.5f; - static float constexpr kFrequency = 440; - static float constexpr kPI = M_PI; - static float constexpr kTwoPi = kPI * 2; - static double constexpr mPhaseIncrement = kFrequency * kTwoPi / (double) kSampleRate; - // Keeps track of where the wave is - float mPhase = 0.0; -}; - -#endif //LOCKSTEP_OBOESINEPLAYER_H diff --git a/app/src/main/cpp/PlaybackEngine.cpp b/app/src/main/cpp/PlaybackEngine.cpp index 4b6dcb1..13470a0 100644 --- a/app/src/main/cpp/PlaybackEngine.cpp +++ b/app/src/main/cpp/PlaybackEngine.cpp @@ -6,12 +6,46 @@ #include "PlaybackEngine.h" #include "logging.h" +#include "mpg123.h" +#include "mp3file.h" +#include -PlaybackEngine::PlaybackEngine(std::string filesDir): mFilesDir(filesDir) { +static inline int readBuffer2(MP3File* mp3) +{ + size_t done = 0; + int err = mpg123_read(mp3->handle, mp3->buffer, mp3->buffer_size, &done); + mp3->leftSamples = done / sizeof(int16_t); + mp3->offset = 0; + return err != MPG123_OK ? 0 : done; +} + +static bool read_mp3(std::string filename, std::vector& samples) { + // TODO: assumes 48000 Hz sampling rate of the file + // note: our resource file is mono (1 channel = 1 samples_per_frame)! + + MP3File *mFile = mp3file_open(filename.c_str(), 0); // MPG123_ENC_FLOAT_32 (but does not seem to work) + bool ok1 = mFile != nullptr; + bool ok2 = readBuffer2(mFile) != 0; // once is enough (maybe the other one needs to pre-fill other buffers, in oboe etc.) + if(ok1 && ok2) { + // num_frames, num_samples, samples_per_frame + samples.resize(mFile->num_samples); + for(int i = 0; i < mFile->num_samples; i++) { + int16_t *src = ((int16_t *) mFile->buffer) + i; + samples[i] = (*src) / 32768.0f; + } + } + if(mFile) + mp3file_delete(mFile); + return ok1 && ok2; +} + +PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(filesDir) { LOGI("PlaybackEngine()"); LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL); // NDK LOG_LEVEL=3 (DEBUG) - mPlayer = new OboeSinePlayer(); + std::vector samples; + read_mp3(mFilesDir + "/" + std::to_string(resid) + ".mp3", samples); + mPlayer = new MixingPlayer(samples); int32_t res = mPlayer->startAudio(); LOGI("startAudio() = %d", res); } diff --git a/app/src/main/cpp/PlaybackEngine.h b/app/src/main/cpp/PlaybackEngine.h index 73b0883..b6bfa3e 100644 --- a/app/src/main/cpp/PlaybackEngine.h +++ b/app/src/main/cpp/PlaybackEngine.h @@ -5,15 +5,15 @@ #ifndef LOCKSTEP_PLAYBACKENGINE_H #define LOCKSTEP_PLAYBACKENGINE_H -#include "OboeSinePlayer.h" +#include "MixingPlayer.h" #include class PlaybackEngine { public: - PlaybackEngine(std::string filesDir); + PlaybackEngine(std::string filesDir, int resid); virtual ~PlaybackEngine(); private: - OboeSinePlayer *mPlayer; + MixingPlayer *mPlayer; std::string mFilesDir; }; diff --git a/app/src/main/cpp/lockstep.cpp b/app/src/main/cpp/lockstep.cpp index 504b95b..e72bffc 100644 --- a/app/src/main/cpp/lockstep.cpp +++ b/app/src/main/cpp/lockstep.cpp @@ -30,13 +30,13 @@ extern "C" { JNIEXPORT jlong JNICALL Java_at_lockstep_pb_PlaybackEngine_native_1createEngine( JNIEnv *env, - jclass /*unused*/, jstring filesDir) { + jclass /*unused*/, jstring filesDir, jint resid) { const char* filesDirTemp = env->GetStringUTFChars(filesDir, NULL); std::string filesDirString(filesDirTemp); env->ReleaseStringUTFChars(filesDir, filesDirTemp); // We use std::nothrow so `new` returns a nullptr if the engine creation fails - auto *engine = new(std::nothrow) PlaybackEngine(filesDirString); + auto *engine = new(std::nothrow) PlaybackEngine(filesDirString, resid); return reinterpret_cast(engine); } diff --git a/app/src/main/java/at/lockstep/app/MainActivity.java b/app/src/main/java/at/lockstep/app/MainActivity.java index 87dfe21..2a04655 100644 --- a/app/src/main/java/at/lockstep/app/MainActivity.java +++ b/app/src/main/java/at/lockstep/app/MainActivity.java @@ -12,7 +12,8 @@ public class MainActivity extends Activity { @Override protected void onResume() { super.onResume(); - PlaybackEngine.create(this); // note: called twice (is permission request causing Activity to go out of focus?) + int resid = at.lockstep.R.raw.track_beat; + PlaybackEngine.create(this, resid); // note: called twice (is permission request causing Activity to go out of focus?) } @Override diff --git a/app/src/main/java/at/lockstep/pb/PlaybackEngine.java b/app/src/main/java/at/lockstep/pb/PlaybackEngine.java index 68b3878..70f5a2e 100644 --- a/app/src/main/java/at/lockstep/pb/PlaybackEngine.java +++ b/app/src/main/java/at/lockstep/pb/PlaybackEngine.java @@ -14,12 +14,19 @@ public class PlaybackEngine { static { System.loadLibrary("lockstep-native"); + // mpg123_init() must be called before createEngine() int ok = native_mpg123_init(); if(ok != MPG123_OK) throw new IllegalStateException("mpg123_init() failed"); } - public static boolean create(Context context) { + /** + * Initialize Oboe library and start playing silence. + * @param context Context for filesystem operations (e.g. Activity) + * @param resid Beat sound (step detection audio-feedback sample) + * @return true if successful + */ + public static boolean create(Context context, int resid) { try { if(!mFilesystemInitialized) { AudioResources.copyRawTracksToFilesystem(context); @@ -33,12 +40,16 @@ public class PlaybackEngine { if (mEngineHandle == 0) { setDefaultStreamValues(context); Log.i("PlaybackEngine", "Hello PlaybackEngine"); - mEngineHandle = native_createEngine(context.getFilesDir().toString()); + mEngineHandle = native_createEngine(context.getFilesDir().toString(), resid); } return (mEngineHandle != 0); } private static void setDefaultStreamValues(Context context) { + // nice-to: + // * re-visit sampling rate and channel count. + // In C++ code, MixingPlayer currently forces both to 48000 and 2 respectively, + // regardless of what Android says would be optimal. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1){ AudioManager myAudioMgr = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); String sampleRateStr = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); @@ -57,7 +68,7 @@ public class PlaybackEngine { mEngineHandle = 0; } - private static native long native_createEngine(String filesDir); + private static native long native_createEngine(String filesDir, int resid); private static native void native_deleteEngine(long engineHandle); private static native void native_setDefaultStreamValues(int sampleRate, int framesPerBurst);