fixup: use stretcher properly, play silence on glitches

app/src/main/cpp/CMakeLists.txt

@@ -37,6 +37,7 @@ add_library(${CMAKE_PROJECT_NAME} SHARED
     jni_stepdetector.cpp
     jni_libpasada.cpp
     LibPasada.cpp
+    TempoController.cpp
         #// jni_logging.cpp // same implemented in jni_libpasada.cpp // JNI_OnLoad and JNI_OnUnload
 )
 

app/src/main/cpp/LibPasada.cpp

@@ -91,12 +91,13 @@ void LibPasada::feedAccel(float x, float y, float z, long long timestamp_nanos)
  * @param fd open read FD (Java retains ownership; do not close until track changes)
  * @param offset start offset within the FD (0 for whole file)
  * @param length byte length from offset ({@code -1} if unknown / to EOF)
+ * @param beats beat times of song in sec
  */
-void LibPasada::play(int fd, long long offset, long long length) {
+void LibPasada::play(int fd, long long offset, long long length, std::vector<double> beats) {
     std::lock_guard<std::mutex> lockState(mtxState);
     state = PLAYING;
     auto *playbackEngine = reinterpret_cast<PlaybackEngine *>(engine);
-    playbackEngine->playMusic(fd, offset, length);
+    playbackEngine->playMusic(fd, beats, offset, length);
 }
 
 /** PLAYING → PAUSED (silent output, graph kept alive). */

app/src/main/cpp/LibPasada.h

@@ -8,6 +8,7 @@
 #include "PasadaPlaybackListener.h"
 #include <string>
 #include <mutex>
+#include <vector>
 
 // JNI helpers for calling into Java code (PasadaPlaybackListener)
 void emitTrackFinished();
@@ -56,8 +57,9 @@ public:
      * @param fd open read FD (Java retains ownership; do not close until track changes)
      * @param offset start offset within the FD (0 for whole file)
      * @param length byte length from offset ({@code -1} if unknown / to EOF)
+     * @param beats beat times of song in sec
      */
-    void play(int fd, long long offset, long long length);
+    void play(int fd, long long offset, long long length, std::vector<double> beats);
     /** PLAYING → PAUSED (silent output, graph kept alive). */
     void pause();
     /** PAUSED → PLAYING (same track, same decode position, same FD). */

app/src/main/cpp/PlaybackEngine.cpp

@@ -92,6 +92,8 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid, PasadaPlaybackLi
     android_fd(0),
     haveTimeRatio(false),
     timeRatio(1.0),
+    haveStretchFactor(false),
+    stretchFactor(1.0),
 
     // these 3 values are preliminary -- will be set from MixingPlayer defaults in the ctor body below
     playbackRate(48000),
@@ -114,6 +116,7 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid, PasadaPlaybackLi
     }
 
     mPlayer = new MixingPlayer(samples);
+    tempoController = new TempoController(&haveStretchFactor, &stretchFactor);
 
     // configure stretcher and start musicFeedThread()
     initRubberBand();
@@ -157,6 +160,13 @@ void PlaybackEngine::initRubberBand() {
 }
 
 void PlaybackEngine::closeRubberBand() {
+    if (mPlayer) {
+        LOGI("PlaybackEngine: mPlayer->stopAudio() ...");
+        mPlayer->stopAudio();
+        LOGI("PlaybackEngine: mPlayer->stopAudio() done.");
+        delete mPlayer;
+        mPlayer = nullptr;
+    }
     if(musicFeed) {
         exitMusicFeedThread.store(true);
         LOGI("PlaybackEngine: musicFeed->join() ...");
@@ -164,7 +174,6 @@ void PlaybackEngine::closeRubberBand() {
         LOGI("PlaybackEngine: musicFeed->join() done.");
         musicFeed = nullptr;
     }
-    closeMusicFile();
 }
 
 void PlaybackEngine::closeMusicFile() {
@@ -209,6 +218,8 @@ void PlaybackEngine::musicFeedThread() {
     std::mt19937 mt(42);
     std::uniform_real_distribution<double> dist(0.98, 1.0);
 */
+    size_t iBeat = 0, numBeats = 0;
+    bool isOnTrackFinished = false;
     LOGI("starting musicFeedThread()");
 
     // strecher num channels: same as output num channels
@@ -243,6 +254,11 @@ void PlaybackEngine::musicFeedThread() {
     // thread 2: polling for decoding more mp3 -> process()  -- getSamplesRequired()
     while(!exitMusicFeedThread.load()) {
 
+        if(isOnTrackFinished) {
+            if(listener) listener->onTrackFinished();
+            isOnTrackFinished = false;
+        }
+
         if(!haveMusicFile.load()) {
             // while no MusicProvider is connected, no samples will be read from 'stretcher'
             // therefore, we do not write any samples into it!
@@ -253,13 +269,21 @@ void PlaybackEngine::musicFeedThread() {
         if(!isSetMusic.load()) {
             mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher, buf_size_samples, numOutChannels.load(), &back_pressure));
             isSetMusic.store(true);
+            numBeats = beatTimesSec.size();
         }
 
-        if(haveTimeRatio.load()) {
+        if(haveTimeRatio.load() || haveStretchFactor.load()) {
+            LOGI("ratio: %.3lf stretch: %.3lf", timeRatio.load(), stretchFactor.load());
+#if 1
+            double ratio = timeRatio.load() * stretchFactor.load();
+#else
             double ratio = timeRatio.load();
+#endif
             stretcher.setTimeRatio(ratio);
-            stretcher.setPitchScale(1.0 / ratio);
+            //stretcher.setPitchScale(1.0 / ratio);
+            stretcher.setPitchScale(1.0 / timeRatio.load());
             haveTimeRatio.store(false);
+            haveStretchFactor.store(false);
         }
 
         // change buffer size, if necessary (changed input channel count)
@@ -325,6 +349,11 @@ void PlaybackEngine::musicFeedThread() {
             continue;
         }
 
+        size_t num_decoded_samples;
+        {
+            std::lock_guard<std::mutex> lock(mMusicLock);
+            if(!haveMusicFile.load()) continue;
+
             if(idebug++ < 10) {
                 pr = std::max(1L, musicFile->rate);
                 loop_delay_us = std::min((size_t) 50000, (size_t) 1000000 * num_samples / pr);
@@ -344,6 +373,11 @@ void PlaybackEngine::musicFeedThread() {
             markerPos.store((musicFile->num_samples - musicFile->remaining_samples) / musicFile->samples_per_frame * musicFile->secs_per_frame);
             duration.store(musicFile->duration);
 
+            if(!beatTimesSec.empty() && iBeat < numBeats && markerPos.load() > beatTimesSec[iBeat]) {
+                iBeat++;
+                if(tempoController) tempoController->onBeat(markerPos.load());
+            }
+
             if(requestSeek.load()) {
                 double pos = seekPos.load();
                 // compute seek target in samples, clip to [0..num_samples]
@@ -369,6 +403,10 @@ void PlaybackEngine::musicFeedThread() {
                     //musicFile->num_bytes = -1; // pretend we can read until the end
                     requestSeek.store(false);
                 /*}*/
+
+                double newMarkerPos = ((musicFile->num_samples - musicFile->remaining_samples) / musicFile->samples_per_frame * musicFile->secs_per_frame);
+
+                if(tempoController) tempoController->seekTo(markerPos.load(), newMarkerPos);
             }
 
             size_t done = 0; // bytes!
@@ -399,6 +437,8 @@ void PlaybackEngine::musicFeedThread() {
                 stretcher.setPitchScale(1.0);
                 stretcher.process(buf_ptr, 0, true); // set end of playback
                 mPlayer->stopAudio();
+                // TODO: raise error
+                //if(listener) listener->onError(1, "mpg123_read() error");
                 continue;
             }
             if(err == MPG123_DONE) {
@@ -407,11 +447,12 @@ void PlaybackEngine::musicFeedThread() {
                 LOGI("finished reading mp3 file (MPG123_DONE)");
                 idebug = 0;
                 closeMusicFile();
-            if(listener) listener->onTrackFinished();
+                //if(listener) listener->onTrackFinished();
+                isOnTrackFinished = true;
                 continue;
             }
 
-        size_t num_decoded_samples = done / sizeof(int16_t) / num_ch_in;
+            num_decoded_samples = done / sizeof(int16_t) / num_ch_in;
             //LOGD("num_decoded_samples = %d", num_decoded_samples);
 
             // * convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
@@ -421,11 +462,17 @@ void PlaybackEngine::musicFeedThread() {
                     buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i * num_ch_in + channel_map[j])) / 32768.0f;
                 }
             }
+        }
 
         //LOGD("calling stretcher.process()");
         stretcher.process(buf_ptr, num_decoded_samples, false);
     }
 
+    if(isOnTrackFinished) {
+        if(listener) listener->onTrackFinished();
+        //isOnTrackFinished = false;
+    }
+
     LOGI("musicFeedThread() exiting ...");
 
     free(buf);
@@ -440,11 +487,13 @@ void PlaybackEngine::pause() {
     LOGI("PlaybackEngine::pause() set isPaused.");
     // next iteration will play silence
     isPaused.store(true);
+    if(tempoController) tempoController->pause();
 }
 
 void PlaybackEngine::resume() {
     // next iteration will continue to decode
     isPaused.store(false);
+    if(tempoController) tempoController->resume();
 }
 
 double PlaybackEngine::getCurrentPosition() {
@@ -462,38 +511,46 @@ void PlaybackEngine::seekTo(double markerPosSec) {
 void PlaybackEngine::stop() {
     LOGI("PlaybackEngine::stop()");
     closeRubberBand();
-    if (mPlayer) {
-        LOGI("PlaybackEngine: mPlayer->stopAudio() ...");
-        mPlayer->stopAudio();
-        LOGI("PlaybackEngine: mPlayer->stopAudio() done.");
-        delete mPlayer;
-        mPlayer = nullptr;
+    {
+        std::lock_guard<std::mutex> lock(mMusicLock);
+        closeMusicFile();
     }
     markerPos.store(0.0);
     duration.store(0.0);
+    if(tempoController) tempoController->pause();
 }
 
 PlaybackEngine::~PlaybackEngine() {
     LOGI("~PlaybackEngine()");
     closeRubberBand();
-    if (mPlayer) {
-        mPlayer->stopAudio();
-        delete mPlayer;
-        mPlayer = nullptr;
+    {
+        std::lock_guard<std::mutex> lock(mMusicLock);
+        closeMusicFile();
     }
+    delete tempoController;
+    tempoController = nullptr;
 }
 
-void PlaybackEngine::playBeat() {
+void PlaybackEngine::playBeat(double t) {
     if(mPlayer) mPlayer->setStartBeat();
+    if(tempoController) tempoController->onStep(t);
 }
 
-void PlaybackEngine::playMusic(int fd, long long offset, long long length) {
+void PlaybackEngine::playMusic(int fd, std::vector<double> beats, long long offset, long long length) {
     if(!mPlayer) return;
-    LOGI("PlaybackEngine::playMusic(fd=%d)", fd);
+    LOGI("PlaybackEngine::playMusic(fd=%d, beats.size=%d)", fd, (int) beats.size());
+
+    {
+        std::lock_guard<std::mutex> lock(mMusicLock);
+
         if (musicFile) {
+            if (fd == android_fd)
+                android_fd = 0; // HACK: avoid closing the 'fd' if re-opening the same file
             // when changing tracks in the UI, close the previous file
             closeMusicFile();
         }
+        beatTimesSec.assign(beats.begin(), beats.end());
+        if(tempoController) tempoController->reset(beatTimesSec);
         android_fd = fd;
         if (fd != 0)
             musicFile.reset(mp3file_open_fd(android_fd, offset, length, 0));
@@ -503,12 +560,14 @@ void PlaybackEngine::playMusic(int fd, long long offset, long long length) {
             numInChannels.store(musicFile->channels);
             haveMusicFile.store(true);
         }
+    }
 
     bool is_finished = (stretcher.available() == -1);
     if(is_finished) {
         LOGE("stretcher.available() == -1, this should not happen anymore with current wiring");
         // so that we may play again after "final chunk"
         closeRubberBand();
+        // do not closeMusicFile() here, it would close the very file we want to be playing
         initRubberBand();
     }
 
@@ -569,6 +628,11 @@ void MusicProvider::onAudioReady(float *data, int32_t frames) {
         // this is an audio glitch
         // TODO: bubble info upwards, in a counter (so we can collect device-specific glitch stats)
         LOGI("stretcher lag: %d requested, %d available", num_frames_requested, num_frames_available);
+        if(num_frames_available == 0) {
+            // at least play silence, avoiding to play garbled stuff
+            memset(data, 0, sizeof(float)*frames*num_ch_out);
+            return;
+        }
     }
     size_t num_frames = std::min(num_frames_available, num_frames_requested);
     stretcher->retrieve(buf_ptr, num_frames);

app/src/main/cpp/PlaybackEngine.h

@@ -11,10 +11,12 @@
 #include "mp3file.h"
 #include "AudioCallback.h"
 #include "PasadaPlaybackListener.h"
+#include "TempoController.h"
 #include <string>
 #include <thread>
 #include <memory>
 #include <atomic>
+#include <vector>
 
 /** Provides music through a regular callback to oboe. Called from separate oboe thread. */
 class MusicProvider : public AudioCallbackProvider {
@@ -45,9 +47,9 @@ public:
     PlaybackEngine(std::string filesDir, int resid, PasadaPlaybackListener *listener = nullptr);
     virtual ~PlaybackEngine();
     /** Play a beat sound. */
-    virtual void playBeat();
+    virtual void playBeat(double t);
     /** pass -1 to length for read until eof. */
-    void playMusic(int fd, long long offset = 0, long long length = -1);
+    void playMusic(int fd, std::vector<double> beats, long long offset = 0, long long length = -1);
     void pause();
     void resume();
     void stop();
@@ -61,8 +63,11 @@ private:
     PasadaPlaybackListener *listener;
     RubberBand::RubberBandStretcher stretcher;
     MixingPlayer *mPlayer;
+    TempoController *tempoController;
     std::string mFilesDir;
+    std::mutex mMusicLock;
     std::unique_ptr<MP3File> musicFile;
+    std::vector<double> beatTimesSec;
     std::atomic<bool> haveMusicFile;
     std::unique_ptr<std::thread> musicFeed;
     std::atomic<bool> exitMusicFeedThread;
@@ -79,7 +84,11 @@ private:
     std::atomic<bool> requestSeek;
     int android_fd;
     std::atomic<bool> haveTimeRatio;
+    /** time ratio of (playback : recording) fps */
     std::atomic<double> timeRatio;
+    std::atomic<bool> haveStretchFactor;
+    /** live stretch factor based on step rate */
+    std::atomic<double> stretchFactor;
     std::atomic<int> playbackRate;
     std::atomic<int> numOutChannels;
     std::atomic<int> numInChannels;

app/src/main/cpp/TempoController.cpp

@@ -0,0 +1,64 @@
+//
+// Created by david on 14.06.2026.
+//
+
+#include "TempoController.h"
+
+TempoController::TempoController(std::atomic<bool> *have_stretch_factor, std::atomic<double> *stretch_factor) :
+    have_stretch_factor(have_stretch_factor),
+    stretch_factor(stretch_factor),
+    lastBeatT(0.0),
+    lastStepT(0.0),
+    dtBeat(0.0),
+    dtStep(0.0)
+{
+    this->have_stretch_factor->store(false);
+    this->stretch_factor->store(1.0);
+}
+
+void TempoController::reset(std::vector<double> beatTimes) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    beatTimesSec.assign(beatTimes.begin(), beatTimes.end());
+    lastBeatT = (0.0);
+    lastStepT = (0.0);
+}
+
+void TempoController::resume() {}
+void TempoController::pause() {}
+
+void TempoController::seekTo(double oldMarkerPosSec, double newMarkerPosSec) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    double dt = newMarkerPosSec - oldMarkerPosSec;
+    lastBeatT += dt;
+}
+
+/**
+ * must be thread-safe since these are called from different threads
+ * @param t marker position in sec of original music time
+ */
+void TempoController::onBeat(double t) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    if(lastBeatT != 0.0) dtBeat = t - lastBeatT;
+    lastBeatT = t;
+    setStretchFactor();
+}
+
+/**
+ * must be thread-safe since these are called from different threads
+ * @param t time in sec of accelerometer timebase
+ */
+void TempoController::onStep(double t) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    if(lastStepT != 0.0) dtStep = t - lastStepT;
+    lastStepT = t;
+    //setStretchFactor();  // one call is enough. otherwise we set twice roughly around the same time.
+}
+
+void TempoController::setStretchFactor() {
+    if(dtBeat == 0.0 || dtStep == 0.0) return;
+    double ratio = dtStep / dtBeat;
+    ratio = std::min(3.3, ratio);
+    ratio = std::max(0.3, ratio);
+    this->have_stretch_factor->store(true);
+    this->stretch_factor->store(ratio);
+}

app/src/main/cpp/TempoController.h

@@ -0,0 +1,42 @@
+//
+// Created by david on 14.06.2026.
+//
+
+#ifndef LOCKSTEP_TEMPOCONTROLLER_H
+#define LOCKSTEP_TEMPOCONTROLLER_H
+
+#include <atomic>
+#include <vector>
+#include <mutex>
+
+class TempoController {
+private:
+    std::mutex mMutex;
+    std::vector<double> beatTimesSec;
+    std::atomic<bool> *have_stretch_factor;
+    std::atomic<double> *stretch_factor;
+    double lastBeatT;
+    double lastStepT;
+    double dtBeat;
+    double dtStep;
+public:
+    TempoController(std::atomic<bool> *have_stretch_factor, std::atomic<double> *stretch_factor);
+    void reset(std::vector<double> beatTimesSec);
+    void resume();
+    void pause();
+    void seekTo(double oldMarkerPosSec, double newMarkerPosSec);
+    /**
+     * must be thread-safe since these are called from different threads
+     * @param t marker position in sec of original music time
+     */
+    void onBeat(double t);
+    /**
+     * must be thread-safe since these are called from different threads
+     * @param t marker position in sec of original music time
+     */
+    void onStep(double t);
+private:
+    void setStretchFactor();
+};
+
+#endif //LOCKSTEP_TEMPOCONTROLLER_H

app/src/main/cpp/jni_libpasada.cpp

@@ -5,6 +5,7 @@
 #include <string>
 #include "LibPasada.h"
 #include "logging.h"
+#include <vector>
 
 static JavaVM* g_vm = nullptr;
 static jobject g_listener = nullptr;      // global ref, or nullptr
@@ -110,13 +111,21 @@ Java_at_lockstep_player_pasada_LibPasada_feedAccel(JNIEnv *env, jclass clazz, jf
 extern "C"
 JNIEXPORT void JNICALL
 Java_at_lockstep_player_pasada_LibPasada_play(JNIEnv *env, jclass clazz, jint fd, jlong offset,
-                                              jlong length) {
+                                              jlong length, jdoubleArray beat_times_sec) {
     LOGD("liblockstep-native jni_libpasada.cpp LibPasada_play()");
     if (g_libpasada == nullptr)
         return;
 
+    std::vector<double> beats;
+    if (beat_times_sec != nullptr) {
+        const jsize n = env->GetArrayLength(beat_times_sec);
+        beats.resize(static_cast<size_t>(n));
+        env->GetDoubleArrayRegion(beat_times_sec, 0, n, beats.data());
+        if (env->ExceptionCheck()) return;
+    }
+
     try {
-        g_libpasada->play(fd, offset, length);
+        g_libpasada->play(fd, offset, length, beats);
     } catch (const std::bad_alloc&) {
         jclass cls = env->FindClass("java/lang/OutOfMemoryError");
         if (cls) env->ThrowNew(cls, "native allocation failed");

app/src/main/cpp/jni_lockstep.cpp

@@ -67,7 +67,8 @@ Java_at_lockstep_pb_PlaybackEngine_native_1playMusic(JNIEnv *env,
                                                      jlong engineHandle,
                                                                   jint fd) {
     auto engine = reinterpret_cast<PlaybackEngine *>(engineHandle);
-    engine->playMusic(fd);
+    std::vector<double> beats;
+    engine->playMusic(fd, beats);
 }
 
 } // extern "C"

app/src/main/java/at/lockstep/player/pasada/LibPasada.java

@@ -1,11 +1,14 @@
 package at.lockstep.player.pasada;
 
 /**
- * JNI entry point for libpasada.
+ * JNI entry point for libpasada. Used by {@link at.lockstep.player.playback.engine.PasadaMusicPlayerEngine};
+ * {@link at.lockstep.player.playback.PlaybackService} still defaults to
+ * {@link at.lockstep.player.playback.engine.ExoPlayerMusicPlayerEngine} until the native library is ready.
  *
  * <p>Native state machine: LOADED → INITIALIZED → PLAYING ↔ PAUSED → FINISHED → STOPPED
  *
- * <p>Call {@link #loadNative()} once before any other method.
+ * <p>Call {@link #loadNative()} once before any other method once the {@code pasada} shared
+ * library is added via CMake (step 3).
  */
 public final class LibPasada {
 
@@ -34,8 +37,14 @@ public final class LibPasada {
      * @param fd open read FD (Java retains ownership; do not close until track changes)
      * @param offset start offset within the FD (0 for whole file)
      * @param length byte length from offset ({@code -1} if unknown / to EOF)
+     * @param beatTimesSec beat times in seconds from track start, or {@code null} when unknown
      */
-    public static native void play(int fd, long offset, long length);
+    public static native void play(int fd, long offset, long length, double[] beatTimesSec);
+
+    /** Same as {@link #play(int, long, long, double[])} with no beat annotation. */
+    public static void play(int fd, long offset, long length) {
+        play(fd, offset, length, null);
+    }
 
     /** PLAYING → PAUSED (silent output, graph kept alive). */
     public static native void pause();