fixup: use stretcher properly, play silence on glitches

app/src/main/cpp/CMakeLists.txt

@@ -37,6 +37,7 @@ add_library(${CMAKE_PROJECT_NAME} SHARED
     jni_stepdetector.cpp
     jni_libpasada.cpp
     LibPasada.cpp
+    TempoController.cpp
         #// jni_logging.cpp // same implemented in jni_libpasada.cpp // JNI_OnLoad and JNI_OnUnload
 )
 

app/src/main/cpp/LibPasada.cpp

@@ -91,12 +91,13 @@ void LibPasada::feedAccel(float x, float y, float z, long long timestamp_nanos)
  * @param fd open read FD (Java retains ownership; do not close until track changes)
  * @param offset start offset within the FD (0 for whole file)
  * @param length byte length from offset ({@code -1} if unknown / to EOF)
+ * @param beats beat times of song in sec
  */
-void LibPasada::play(int fd, long long offset, long long length) {
+void LibPasada::play(int fd, long long offset, long long length, std::vector<double> beats) {
     std::lock_guard<std::mutex> lockState(mtxState);
     state = PLAYING;
     auto *playbackEngine = reinterpret_cast<PlaybackEngine *>(engine);
-    playbackEngine->playMusic(fd, offset, length);
+    playbackEngine->playMusic(fd, beats, offset, length);
 }
 
 /** PLAYING → PAUSED (silent output, graph kept alive). */

app/src/main/cpp/LibPasada.h

@@ -8,6 +8,7 @@
 #include "PasadaPlaybackListener.h"
 #include <string>
 #include <mutex>
+#include <vector>
 
 // JNI helpers for calling into Java code (PasadaPlaybackListener)
 void emitTrackFinished();
@@ -56,8 +57,9 @@ public:
      * @param fd open read FD (Java retains ownership; do not close until track changes)
      * @param offset start offset within the FD (0 for whole file)
      * @param length byte length from offset ({@code -1} if unknown / to EOF)
+     * @param beats beat times of song in sec
      */
-    void play(int fd, long long offset, long long length);
+    void play(int fd, long long offset, long long length, std::vector<double> beats);
     /** PLAYING → PAUSED (silent output, graph kept alive). */
     void pause();
     /** PAUSED → PLAYING (same track, same decode position, same FD). */

app/src/main/cpp/PlaybackEngine.cpp

@@ -92,6 +92,8 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid, PasadaPlaybackLi
     android_fd(0),
     haveTimeRatio(false),
     timeRatio(1.0),
+    haveStretchFactor(false),
+    stretchFactor(1.0),
 
     // these 3 values are preliminary -- will be set from MixingPlayer defaults in the ctor body below
     playbackRate(48000),
@@ -114,6 +116,7 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid, PasadaPlaybackLi
     }
 
     mPlayer = new MixingPlayer(samples);
+    tempoController = new TempoController(&haveStretchFactor, &stretchFactor);
 
     // configure stretcher and start musicFeedThread()
     initRubberBand();
@@ -157,6 +160,13 @@ void PlaybackEngine::initRubberBand() {
 }
 
 void PlaybackEngine::closeRubberBand() {
+    if (mPlayer) {
+        LOGI("PlaybackEngine: mPlayer->stopAudio() ...");
+        mPlayer->stopAudio();
+        LOGI("PlaybackEngine: mPlayer->stopAudio() done.");
+        delete mPlayer;
+        mPlayer = nullptr;
+    }
     if(musicFeed) {
         exitMusicFeedThread.store(true);
         LOGI("PlaybackEngine: musicFeed->join() ...");
@@ -164,7 +174,6 @@ void PlaybackEngine::closeRubberBand() {
         LOGI("PlaybackEngine: musicFeed->join() done.");
         musicFeed = nullptr;
     }
-    closeMusicFile();
 }
 
 void PlaybackEngine::closeMusicFile() {
@@ -209,6 +218,8 @@ void PlaybackEngine::musicFeedThread() {
     std::mt19937 mt(42);
     std::uniform_real_distribution<double> dist(0.98, 1.0);
 */
+    size_t iBeat = 0, numBeats = 0;
+    bool isOnTrackFinished = false;
     LOGI("starting musicFeedThread()");
 
     // strecher num channels: same as output num channels
@@ -243,6 +254,11 @@ void PlaybackEngine::musicFeedThread() {
     // thread 2: polling for decoding more mp3 -> process()  -- getSamplesRequired()
     while(!exitMusicFeedThread.load()) {
 
+        if(isOnTrackFinished) {
+            if(listener) listener->onTrackFinished();
+            isOnTrackFinished = false;
+        }
+
         if(!haveMusicFile.load()) {
             // while no MusicProvider is connected, no samples will be read from 'stretcher'
             // therefore, we do not write any samples into it!
@@ -253,13 +269,21 @@ void PlaybackEngine::musicFeedThread() {
         if(!isSetMusic.load()) {
             mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher, buf_size_samples, numOutChannels.load(), &back_pressure));
             isSetMusic.store(true);
+            numBeats = beatTimesSec.size();
         }
 
-        if(haveTimeRatio.load()) {
+        if(haveTimeRatio.load() || haveStretchFactor.load()) {
+            LOGI("ratio: %.3lf stretch: %.3lf", timeRatio.load(), stretchFactor.load());
+#if 1
+            double ratio = timeRatio.load() * stretchFactor.load();
+#else
             double ratio = timeRatio.load();
+#endif
             stretcher.setTimeRatio(ratio);
-            stretcher.setPitchScale(1.0 / ratio);
+            //stretcher.setPitchScale(1.0 / ratio);
+            stretcher.setPitchScale(1.0 / timeRatio.load());
             haveTimeRatio.store(false);
+            haveStretchFactor.store(false);
         }
 
         // change buffer size, if necessary (changed input channel count)
@@ -325,100 +349,118 @@ void PlaybackEngine::musicFeedThread() {
             continue;
         }
 
-        if(idebug++ < 10) {
-            pr = std::max(1L, musicFile->rate);
-            loop_delay_us = std::min((size_t) 50000, (size_t) 1000000 * num_samples / pr);
-            LOGI("feed %d music samples", num_samples);
-            // feed 1024 music samples
-            // => stretcher is asking for 1024 = getSamplesRequired()
-            // ca. 21 ms
+        size_t num_decoded_samples;
+        {
+            std::lock_guard<std::mutex> lock(mMusicLock);
+            if(!haveMusicFile.load()) continue;
+
+            if(idebug++ < 10) {
+                pr = std::max(1L, musicFile->rate);
+                loop_delay_us = std::min((size_t) 50000, (size_t) 1000000 * num_samples / pr);
+                LOGI("feed %d music samples", num_samples);
+                // feed 1024 music samples
+                // => stretcher is asking for 1024 = getSamplesRequired()
+                // ca. 21 ms
 
 /*
-            int64_t a = mpg123_seek(musicFile->handle, 0, SEEK_CUR);
-            int64_t b = mpg123_tell(musicFile->handle);
-            LOGI("  checking mpg123_seek(): len=%lld seek=%lld tell=%lld",
-                 (long long)musicFile->num_samples, (long long)a, (long long)b);
+                int64_t a = mpg123_seek(musicFile->handle, 0, SEEK_CUR);
+                int64_t b = mpg123_tell(musicFile->handle);
+                LOGI("  checking mpg123_seek(): len=%lld seek=%lld tell=%lld",
+                     (long long)musicFile->num_samples, (long long)a, (long long)b);
 */
-        }
+            }
 
-        markerPos.store((musicFile->num_samples - musicFile->remaining_samples) / musicFile->samples_per_frame * musicFile->secs_per_frame);
-        duration.store(musicFile->duration);
+            markerPos.store((musicFile->num_samples - musicFile->remaining_samples) / musicFile->samples_per_frame * musicFile->secs_per_frame);
+            duration.store(musicFile->duration);
 
-        if(requestSeek.load()) {
-            double pos = seekPos.load();
-            // compute seek target in samples, clip to [0..num_samples]
-            auto seek_samples = static_cast<long>(pos * musicFile->samples_per_frame / musicFile->secs_per_frame);
-            LOGI("seek to %.3lf sec = %lld samples", pos, (long long) seek_samples);
-            seek_samples = std::max(seek_samples, 0L);
-            seek_samples = std::min(seek_samples, musicFile->num_samples);
-            // perform the seek
-            off_t seekResult = mp3file_seek(musicFile.get(), seek_samples);
-            idebug = 0;
-            /*if(seekResult > seek_samples) {
-                LOGE("error seeking in mp3 file: mpg123_seek(handle, %ld, SEEK_SET)=%lld", seek_samples, (long long) seekResult);
-                seekPos.store(pos * dist(mt)); // randomize seek target a bit
-            } else {*/
-                // update structure
-                // compute seek delta, to update byte 'offset'
-                //auto seek_samples_delta = seekResult - (musicFile->num_samples - musicFile->remaining_samples);
-                //auto seek_bytes_delta = seek_samples_delta * num_ch_in * sizeof(int16_t);
-                musicFile->remaining_samples = static_cast<int>(musicFile->num_samples - seekResult);
-                //musicFile->offset += seek_bytes_delta;
-                musicFile->offset = lseek(musicFile->android_fd, 0, SEEK_CUR);
-                // it is not possible to obtain bytes offset reliably??
-                //musicFile->num_bytes = -1; // pretend we can read until the end
-                requestSeek.store(false);
-            /*}*/
-        }
+            if(!beatTimesSec.empty() && iBeat < numBeats && markerPos.load() > beatTimesSec[iBeat]) {
+                iBeat++;
+                if(tempoController) tempoController->onBeat(markerPos.load());
+            }
 
-        size_t done = 0; // bytes!
-        size_t read_size_bytes = std::min(num_samples * num_ch_in * sizeof(int16_t), cbuf_size_bytes);
-        /*if (musicFile->offset != 0 && musicFile->num_bytes != -1) { read_size_bytes = std::min(read_size_bytes_calc, (size_t) musicFile->num_bytes - musicFile->offset); }*/
-        //if (idebug < 10) LOGI("mft rsbc=%lld rsb=%lld nb=%lld o=%lld", (long long) read_size_bytes_calc, (long long) read_size_bytes, (long long) musicFile->num_bytes, (long long) musicFile->offset);
-        int err = mpg123_read(musicFile->handle, cbuf, read_size_bytes, &done);
-        //if (idebug < 10) LOGI("mft done=%lld err=%lld small=%d", (long long) done, (long long) err, read_size_bytes < read_size_bytes_calc);
-        /*if (musicFile->offset != 0 && musicFile->num_bytes != -1 && err == MPG123_OK && read_size_bytes < read_size_bytes_calc) {
-            LOGW("out of bytes, finished playing");
-            err = MPG123_DONE;
-        }*/
-        musicFile->remaining_samples -= done / num_ch_in / sizeof(int16_t);
-        //musicFile->offset += done;
-        musicFile->offset = lseek(musicFile->android_fd, 0, SEEK_CUR);
-        if(musicFile->offset >= musicFile->num_bytes) {
-            // TODO: this is inaccurate. either we do not decode everything, or we overrun past the end
-            // TODO: maybe decode as a "stream" instead.
-            LOGW("out of bytes, finished playing");
-            err = MPG123_DONE;
-        }
-        if (err != MPG123_OK && err != MPG123_DONE) {
-            // error!
-            LOGE("error reading mp3 file: mpg123_read() err=%d done=%d", err, done);
-            // next iteration will play silence
-            closeMusicFile();
-            stretcher.setTimeRatio(1.0); // buffer size for playing silence is computed from 'playbackRate', so reset timeRatio
-            stretcher.setPitchScale(1.0);
-            stretcher.process(buf_ptr, 0, true); // set end of playback
-            mPlayer->stopAudio();
-            continue;
-        }
-        if(err == MPG123_DONE) {
-            // next iteration will play silence
-            // we keep Stretcher and Oboe alive
-            LOGI("finished reading mp3 file (MPG123_DONE)");
-            idebug = 0;
-            closeMusicFile();
-            if(listener) listener->onTrackFinished();
-            continue;
-        }
+            if(requestSeek.load()) {
+                double pos = seekPos.load();
+                // compute seek target in samples, clip to [0..num_samples]
+                auto seek_samples = static_cast<long>(pos * musicFile->samples_per_frame / musicFile->secs_per_frame);
+                LOGI("seek to %.3lf sec = %lld samples", pos, (long long) seek_samples);
+                seek_samples = std::max(seek_samples, 0L);
+                seek_samples = std::min(seek_samples, musicFile->num_samples);
+                // perform the seek
+                off_t seekResult = mp3file_seek(musicFile.get(), seek_samples);
+                idebug = 0;
+                /*if(seekResult > seek_samples) {
+                    LOGE("error seeking in mp3 file: mpg123_seek(handle, %ld, SEEK_SET)=%lld", seek_samples, (long long) seekResult);
+                    seekPos.store(pos * dist(mt)); // randomize seek target a bit
+                } else {*/
+                    // update structure
+                    // compute seek delta, to update byte 'offset'
+                    //auto seek_samples_delta = seekResult - (musicFile->num_samples - musicFile->remaining_samples);
+                    //auto seek_bytes_delta = seek_samples_delta * num_ch_in * sizeof(int16_t);
+                    musicFile->remaining_samples = static_cast<int>(musicFile->num_samples - seekResult);
+                    //musicFile->offset += seek_bytes_delta;
+                    musicFile->offset = lseek(musicFile->android_fd, 0, SEEK_CUR);
+                    // it is not possible to obtain bytes offset reliably??
+                    //musicFile->num_bytes = -1; // pretend we can read until the end
+                    requestSeek.store(false);
+                /*}*/
 
-        size_t num_decoded_samples = done / sizeof(int16_t) / num_ch_in;
-        //LOGD("num_decoded_samples = %d", num_decoded_samples);
+                double newMarkerPos = ((musicFile->num_samples - musicFile->remaining_samples) / musicFile->samples_per_frame * musicFile->secs_per_frame);
 
-        // * convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
-        // * map input to output channels
-        for(size_t i = 0; i < num_decoded_samples; i++) {
-            for(size_t j = 0; j < num_ch_out; j++) {
-                buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i * num_ch_in + channel_map[j])) / 32768.0f;
+                if(tempoController) tempoController->seekTo(markerPos.load(), newMarkerPos);
+            }
+
+            size_t done = 0; // bytes!
+            size_t read_size_bytes = std::min(num_samples * num_ch_in * sizeof(int16_t), cbuf_size_bytes);
+            /*if (musicFile->offset != 0 && musicFile->num_bytes != -1) { read_size_bytes = std::min(read_size_bytes_calc, (size_t) musicFile->num_bytes - musicFile->offset); }*/
+            //if (idebug < 10) LOGI("mft rsbc=%lld rsb=%lld nb=%lld o=%lld", (long long) read_size_bytes_calc, (long long) read_size_bytes, (long long) musicFile->num_bytes, (long long) musicFile->offset);
+            int err = mpg123_read(musicFile->handle, cbuf, read_size_bytes, &done);
+            //if (idebug < 10) LOGI("mft done=%lld err=%lld small=%d", (long long) done, (long long) err, read_size_bytes < read_size_bytes_calc);
+            /*if (musicFile->offset != 0 && musicFile->num_bytes != -1 && err == MPG123_OK && read_size_bytes < read_size_bytes_calc) {
+                LOGW("out of bytes, finished playing");
+                err = MPG123_DONE;
+            }*/
+            musicFile->remaining_samples -= done / num_ch_in / sizeof(int16_t);
+            //musicFile->offset += done;
+            musicFile->offset = lseek(musicFile->android_fd, 0, SEEK_CUR);
+            if(musicFile->offset >= musicFile->num_bytes) {
+                // TODO: this is inaccurate. either we do not decode everything, or we overrun past the end
+                // TODO: maybe decode as a "stream" instead.
+                LOGW("out of bytes, finished playing");
+                err = MPG123_DONE;
+            }
+            if (err != MPG123_OK && err != MPG123_DONE) {
+                // error!
+                LOGE("error reading mp3 file: mpg123_read() err=%d done=%d", err, done);
+                // next iteration will play silence
+                closeMusicFile();
+                stretcher.setTimeRatio(1.0); // buffer size for playing silence is computed from 'playbackRate', so reset timeRatio
+                stretcher.setPitchScale(1.0);
+                stretcher.process(buf_ptr, 0, true); // set end of playback
+                mPlayer->stopAudio();
+                // TODO: raise error
+                //if(listener) listener->onError(1, "mpg123_read() error");
+                continue;
+            }
+            if(err == MPG123_DONE) {
+                // next iteration will play silence
+                // we keep Stretcher and Oboe alive
+                LOGI("finished reading mp3 file (MPG123_DONE)");
+                idebug = 0;
+                closeMusicFile();
+                //if(listener) listener->onTrackFinished();
+                isOnTrackFinished = true;
+                continue;
+            }
+
+            num_decoded_samples = done / sizeof(int16_t) / num_ch_in;
+            //LOGD("num_decoded_samples = %d", num_decoded_samples);
+
+            // * convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
+            // * map input to output channels
+            for(size_t i = 0; i < num_decoded_samples; i++) {
+                for(size_t j = 0; j < num_ch_out; j++) {
+                    buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i * num_ch_in + channel_map[j])) / 32768.0f;
+                }
             }
         }
 
@@ -426,6 +468,11 @@ void PlaybackEngine::musicFeedThread() {
         stretcher.process(buf_ptr, num_decoded_samples, false);
     }
 
+    if(isOnTrackFinished) {
+        if(listener) listener->onTrackFinished();
+        //isOnTrackFinished = false;
+    }
+
     LOGI("musicFeedThread() exiting ...");
 
     free(buf);
@@ -440,11 +487,13 @@ void PlaybackEngine::pause() {
     LOGI("PlaybackEngine::pause() set isPaused.");
     // next iteration will play silence
     isPaused.store(true);
+    if(tempoController) tempoController->pause();
 }
 
 void PlaybackEngine::resume() {
     // next iteration will continue to decode
     isPaused.store(false);
+    if(tempoController) tempoController->resume();
 }
 
 double PlaybackEngine::getCurrentPosition() {
@@ -462,46 +511,55 @@ void PlaybackEngine::seekTo(double markerPosSec) {
 void PlaybackEngine::stop() {
     LOGI("PlaybackEngine::stop()");
     closeRubberBand();
-    if (mPlayer) {
-        LOGI("PlaybackEngine: mPlayer->stopAudio() ...");
-        mPlayer->stopAudio();
-        LOGI("PlaybackEngine: mPlayer->stopAudio() done.");
-        delete mPlayer;
-        mPlayer = nullptr;
+    {
+        std::lock_guard<std::mutex> lock(mMusicLock);
+        closeMusicFile();
     }
     markerPos.store(0.0);
     duration.store(0.0);
+    if(tempoController) tempoController->pause();
 }
 
 PlaybackEngine::~PlaybackEngine() {
     LOGI("~PlaybackEngine()");
     closeRubberBand();
-    if (mPlayer) {
-        mPlayer->stopAudio();
-        delete mPlayer;
-        mPlayer = nullptr;
-    }
-}
-
-void PlaybackEngine::playBeat() {
-    if(mPlayer) mPlayer->setStartBeat();
-}
-
-void PlaybackEngine::playMusic(int fd, long long offset, long long length) {
-    if(!mPlayer) return;
-    LOGI("PlaybackEngine::playMusic(fd=%d)", fd);
-    if(musicFile) {
-        // when changing tracks in the UI, close the previous file
+    {
+        std::lock_guard<std::mutex> lock(mMusicLock);
         closeMusicFile();
     }
-    android_fd = fd;
-    if(fd != 0)
-        musicFile.reset(mp3file_open_fd(android_fd, offset, length, 0));
-    if(musicFile) {
-        timeRatio.store(((double) playbackRate.load()) / ((double) musicFile->rate));
-        haveTimeRatio.store(true);
-        numInChannels.store(musicFile->channels);
-        haveMusicFile.store(true);
+    delete tempoController;
+    tempoController = nullptr;
+}
+
+void PlaybackEngine::playBeat(double t) {
+    if(mPlayer) mPlayer->setStartBeat();
+    if(tempoController) tempoController->onStep(t);
+}
+
+void PlaybackEngine::playMusic(int fd, std::vector<double> beats, long long offset, long long length) {
+    if(!mPlayer) return;
+    LOGI("PlaybackEngine::playMusic(fd=%d, beats.size=%d)", fd, (int) beats.size());
+
+    {
+        std::lock_guard<std::mutex> lock(mMusicLock);
+
+        if (musicFile) {
+            if (fd == android_fd)
+                android_fd = 0; // HACK: avoid closing the 'fd' if re-opening the same file
+            // when changing tracks in the UI, close the previous file
+            closeMusicFile();
+        }
+        beatTimesSec.assign(beats.begin(), beats.end());
+        if(tempoController) tempoController->reset(beatTimesSec);
+        android_fd = fd;
+        if (fd != 0)
+            musicFile.reset(mp3file_open_fd(android_fd, offset, length, 0));
+        if (musicFile) {
+            timeRatio.store(((double) playbackRate.load()) / ((double) musicFile->rate));
+            haveTimeRatio.store(true);
+            numInChannels.store(musicFile->channels);
+            haveMusicFile.store(true);
+        }
     }
 
     bool is_finished = (stretcher.available() == -1);
@@ -509,6 +567,7 @@ void PlaybackEngine::playMusic(int fd, long long offset, long long length) {
         LOGE("stretcher.available() == -1, this should not happen anymore with current wiring");
         // so that we may play again after "final chunk"
         closeRubberBand();
+        // do not closeMusicFile() here, it would close the very file we want to be playing
         initRubberBand();
     }
 
@@ -569,6 +628,11 @@ void MusicProvider::onAudioReady(float *data, int32_t frames) {
         // this is an audio glitch
         // TODO: bubble info upwards, in a counter (so we can collect device-specific glitch stats)
         LOGI("stretcher lag: %d requested, %d available", num_frames_requested, num_frames_available);
+        if(num_frames_available == 0) {
+            // at least play silence, avoiding to play garbled stuff
+            memset(data, 0, sizeof(float)*frames*num_ch_out);
+            return;
+        }
     }
     size_t num_frames = std::min(num_frames_available, num_frames_requested);
     stretcher->retrieve(buf_ptr, num_frames);

app/src/main/cpp/PlaybackEngine.h

@@ -11,10 +11,12 @@
 #include "mp3file.h"
 #include "AudioCallback.h"
 #include "PasadaPlaybackListener.h"
+#include "TempoController.h"
 #include <string>
 #include <thread>
 #include <memory>
 #include <atomic>
+#include <vector>
 
 /** Provides music through a regular callback to oboe. Called from separate oboe thread. */
 class MusicProvider : public AudioCallbackProvider {
@@ -45,9 +47,9 @@ public:
     PlaybackEngine(std::string filesDir, int resid, PasadaPlaybackListener *listener = nullptr);
     virtual ~PlaybackEngine();
     /** Play a beat sound. */
-    virtual void playBeat();
+    virtual void playBeat(double t);
     /** pass -1 to length for read until eof. */
-    void playMusic(int fd, long long offset = 0, long long length = -1);
+    void playMusic(int fd, std::vector<double> beats, long long offset = 0, long long length = -1);
     void pause();
     void resume();
     void stop();
@@ -61,8 +63,11 @@ private:
     PasadaPlaybackListener *listener;
     RubberBand::RubberBandStretcher stretcher;
     MixingPlayer *mPlayer;
+    TempoController *tempoController;
     std::string mFilesDir;
+    std::mutex mMusicLock;
     std::unique_ptr<MP3File> musicFile;
+    std::vector<double> beatTimesSec;
     std::atomic<bool> haveMusicFile;
     std::unique_ptr<std::thread> musicFeed;
     std::atomic<bool> exitMusicFeedThread;
@@ -79,7 +84,11 @@ private:
     std::atomic<bool> requestSeek;
     int android_fd;
     std::atomic<bool> haveTimeRatio;
+    /** time ratio of (playback : recording) fps */
     std::atomic<double> timeRatio;
+    std::atomic<bool> haveStretchFactor;
+    /** live stretch factor based on step rate */
+    std::atomic<double> stretchFactor;
     std::atomic<int> playbackRate;
     std::atomic<int> numOutChannels;
     std::atomic<int> numInChannels;

app/src/main/cpp/TempoController.cpp

@@ -0,0 +1,64 @@
+//
+// Created by david on 14.06.2026.
+//
+
+#include "TempoController.h"
+
+TempoController::TempoController(std::atomic<bool> *have_stretch_factor, std::atomic<double> *stretch_factor) :
+    have_stretch_factor(have_stretch_factor),
+    stretch_factor(stretch_factor),
+    lastBeatT(0.0),
+    lastStepT(0.0),
+    dtBeat(0.0),
+    dtStep(0.0)
+{
+    this->have_stretch_factor->store(false);
+    this->stretch_factor->store(1.0);
+}
+
+void TempoController::reset(std::vector<double> beatTimes) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    beatTimesSec.assign(beatTimes.begin(), beatTimes.end());
+    lastBeatT = (0.0);
+    lastStepT = (0.0);
+}
+
+void TempoController::resume() {}
+void TempoController::pause() {}
+
+void TempoController::seekTo(double oldMarkerPosSec, double newMarkerPosSec) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    double dt = newMarkerPosSec - oldMarkerPosSec;
+    lastBeatT += dt;
+}
+
+/**
+ * must be thread-safe since these are called from different threads
+ * @param t marker position in sec of original music time
+ */
+void TempoController::onBeat(double t) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    if(lastBeatT != 0.0) dtBeat = t - lastBeatT;
+    lastBeatT = t;
+    setStretchFactor();
+}
+
+/**
+ * must be thread-safe since these are called from different threads
+ * @param t time in sec of accelerometer timebase
+ */
+void TempoController::onStep(double t) {
+    std::lock_guard<std::mutex> lock(mMutex);
+    if(lastStepT != 0.0) dtStep = t - lastStepT;
+    lastStepT = t;
+    //setStretchFactor();  // one call is enough. otherwise we set twice roughly around the same time.
+}
+
+void TempoController::setStretchFactor() {
+    if(dtBeat == 0.0 || dtStep == 0.0) return;
+    double ratio = dtStep / dtBeat;
+    ratio = std::min(3.3, ratio);
+    ratio = std::max(0.3, ratio);
+    this->have_stretch_factor->store(true);
+    this->stretch_factor->store(ratio);
+}

app/src/main/cpp/TempoController.h

@@ -0,0 +1,42 @@
+//
+// Created by david on 14.06.2026.
+//
+
+#ifndef LOCKSTEP_TEMPOCONTROLLER_H
+#define LOCKSTEP_TEMPOCONTROLLER_H
+
+#include <atomic>
+#include <vector>
+#include <mutex>
+
+class TempoController {
+private:
+    std::mutex mMutex;
+    std::vector<double> beatTimesSec;
+    std::atomic<bool> *have_stretch_factor;
+    std::atomic<double> *stretch_factor;
+    double lastBeatT;
+    double lastStepT;
+    double dtBeat;
+    double dtStep;
+public:
+    TempoController(std::atomic<bool> *have_stretch_factor, std::atomic<double> *stretch_factor);
+    void reset(std::vector<double> beatTimesSec);
+    void resume();
+    void pause();
+    void seekTo(double oldMarkerPosSec, double newMarkerPosSec);
+    /**
+     * must be thread-safe since these are called from different threads
+     * @param t marker position in sec of original music time
+     */
+    void onBeat(double t);
+    /**
+     * must be thread-safe since these are called from different threads
+     * @param t marker position in sec of original music time
+     */
+    void onStep(double t);
+private:
+    void setStretchFactor();
+};
+
+#endif //LOCKSTEP_TEMPOCONTROLLER_H

app/src/main/cpp/jni_libpasada.cpp

@@ -5,6 +5,7 @@
 #include <string>
 #include "LibPasada.h"
 #include "logging.h"
+#include <vector>
 
 static JavaVM* g_vm = nullptr;
 static jobject g_listener = nullptr;      // global ref, or nullptr
@@ -110,13 +111,21 @@ Java_at_lockstep_player_pasada_LibPasada_feedAccel(JNIEnv *env, jclass clazz, jf
 extern "C"
 JNIEXPORT void JNICALL
 Java_at_lockstep_player_pasada_LibPasada_play(JNIEnv *env, jclass clazz, jint fd, jlong offset,
-                                              jlong length) {
+                                              jlong length, jdoubleArray beat_times_sec) {
     LOGD("liblockstep-native jni_libpasada.cpp LibPasada_play()");
     if (g_libpasada == nullptr)
         return;
 
+    std::vector<double> beats;
+    if (beat_times_sec != nullptr) {
+        const jsize n = env->GetArrayLength(beat_times_sec);
+        beats.resize(static_cast<size_t>(n));
+        env->GetDoubleArrayRegion(beat_times_sec, 0, n, beats.data());
+        if (env->ExceptionCheck()) return;
+    }
+
     try {
-        g_libpasada->play(fd, offset, length);
+        g_libpasada->play(fd, offset, length, beats);
     } catch (const std::bad_alloc&) {
         jclass cls = env->FindClass("java/lang/OutOfMemoryError");
         if (cls) env->ThrowNew(cls, "native allocation failed");

app/src/main/cpp/jni_lockstep.cpp

@@ -67,7 +67,8 @@ Java_at_lockstep_pb_PlaybackEngine_native_1playMusic(JNIEnv *env,
                                                      jlong engineHandle,
                                                                   jint fd) {
     auto engine = reinterpret_cast<PlaybackEngine *>(engineHandle);
-    engine->playMusic(fd);
+    std::vector<double> beats;
+    engine->playMusic(fd, beats);
 }
 
 } // extern "C"

app/src/main/java/at/lockstep/player/pasada/LibPasada.java

@@ -1,11 +1,14 @@
 package at.lockstep.player.pasada;
 
 /**
- * JNI entry point for libpasada.
+ * JNI entry point for libpasada. Used by {@link at.lockstep.player.playback.engine.PasadaMusicPlayerEngine};
+ * {@link at.lockstep.player.playback.PlaybackService} still defaults to
+ * {@link at.lockstep.player.playback.engine.ExoPlayerMusicPlayerEngine} until the native library is ready.
  *
  * <p>Native state machine: LOADED → INITIALIZED → PLAYING ↔ PAUSED → FINISHED → STOPPED
  *
- * <p>Call {@link #loadNative()} once before any other method.
+ * <p>Call {@link #loadNative()} once before any other method once the {@code pasada} shared
+ * library is added via CMake (step 3).
  */
 public final class LibPasada {
 
@@ -34,8 +37,14 @@ public final class LibPasada {
      * @param fd open read FD (Java retains ownership; do not close until track changes)
      * @param offset start offset within the FD (0 for whole file)
      * @param length byte length from offset ({@code -1} if unknown / to EOF)
+     * @param beatTimesSec beat times in seconds from track start, or {@code null} when unknown
      */
-    public static native void play(int fd, long offset, long length);
+    public static native void play(int fd, long offset, long length, double[] beatTimesSec);
+
+    /** Same as {@link #play(int, long, long, double[])} with no beat annotation. */
+    public static void play(int fd, long offset, long length) {
+        play(fd, offset, length, null);
+    }
 
     /** PLAYING → PAUSED (silent output, graph kept alive). */
     public static native void pause();