david/lockstep
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: play music through librubberband

Browse Source
This commit is contained in:
David Madl
2026-03-20 23:17:34 +01:00
parent 5b26203533
commit b14ea02694
8 changed files with 314 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app/src/main/cpp/AudioCallback.h Normal file
View File

@@ -0,0 +1,20 @@
//
// Created by david on 20.03.2026.
//
#ifndef LOCKSTEP_AUDIOCALLBACK_H
#define LOCKSTEP_AUDIOCALLBACK_H
#include <cstdint>
/**
* Provides audio through a regular callback to oboe.
*/
class AudioCallbackProvider {
public:
virtual ~AudioCallbackProvider() {}
/** in current impl, this passes a buffer where data may already live. the provider may add to it and re-normalize to [-1.0, 1.0]. */
virtual void onAudioReady(float *data, int32_t frames) {}
};
#endif //LOCKSTEP_AUDIOCALLBACK_H

1
app/src/main/cpp/CMakeLists.txt
View File

@@ -51,6 +51,7 @@ set_target_properties(mpg123 PROPERTIES IMPORTED_LOCATION
include_directories(${mpg123_DIR}/lib/${ANDROID_ABI}/include)
target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE libpasada/pasada-lib/include)
target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE librubberband/rubberband)
# Specifies libraries CMake should link to your target library. You
# can link libraries from various origins, such as libraries defined in this

21
app/src/main/cpp/MixingPlayer.h
View File

@@ -6,7 +6,10 @@
#define LOCKSTEP_MIXINGPLAYER_H
#include <oboe/Oboe.h>
#include <math.h>
#include <cmath>
#include <atomic>
#include "AudioCallback.h"
using namespace oboe;
/**
@@ -20,7 +23,7 @@ protected:
std::atomic<int> startBeat;
int numBeatsPlaying;
public:
explicit MixingPlayer(std::vector<float> beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0) {}
explicit MixingPlayer(std::vector<float> beatSound) : beatSound(beatSound), startBeat(0), numBeatsPlaying(0), mHaveMusic(false) {}
virtual ~MixingPlayer() = default;
@@ -63,6 +66,12 @@ public:
}
}
void setMusic(std::shared_ptr<AudioCallbackProvider> cb) {
std::lock_guard<std::mutex> lock(mLock);
mMusic = std::move(cb);
mHaveMusic.store((bool) mMusic);
}
oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override {
// fetch startBeat register
int isStartBeat = startBeat.load();
@@ -78,7 +87,7 @@ public:
int N = (int) beatSound.size();
for (int i = 0; i < numFrames; i++) {
float sample = 0.0;
float norm = (float) numBeatsPlaying;
float norm = (float) (std::max(numBeatsPlaying, 1));
for (int k = 0; k < numBeatsPlaying; k++) {
sample += beatSound[beatIdx[k]];
beatIdx[k] += 1;
@@ -102,12 +111,18 @@ public:
}
}
if(mHaveMusic.load()) {
mMusic->onAudioReady(floatData, numFrames);
}
return oboe::DataCallbackResult::Continue;
}
private:
std::mutex mLock;
std::shared_ptr<oboe::AudioStream> mStream;
std::shared_ptr<AudioCallbackProvider> mMusic;
std::atomic<bool> mHaveMusic;
// Stream params
static int constexpr kChannelCount = 2;

220
app/src/main/cpp/PlaybackEngine.cpp
View File

@@ -8,7 +8,9 @@
#include "logging.h"
#include "mpg123.h"
#include "mp3file.h"
#include <memory>
#include <vector>
#include <chrono>
/**
* Read samples from the next mp3-frame into the struct MP3File's buffer.
@@ -52,7 +54,34 @@ static bool read_mp3(std::string filename, std::vector<float>& samples) {
return ok1 && ok2;
}
PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(filesDir) {
struct RbLogger : public RubberBand::RubberBandStretcher::Logger {
virtual void log(const char *s) {
LOGI("%s", s);
}
virtual void log(const char *s, double val) {
LOGI("%s (%lf)", s, val);
}
virtual void log(const char *s, double val1, double val2) {
LOGI("%s (%lf, %lf)", s, val1, val2);
}
virtual ~RbLogger() { }
};
PlaybackEngine::PlaybackEngine(std::string filesDir, int resid):
stretcher(
/* sampleRate: */ 48000,
/* channels: */ 2,
/* logger: */ std::make_shared<RbLogger>(),
/* options: */
RubberBand::RubberBandStretcher::OptionEngineFaster |
RubberBand::RubberBandStretcher::OptionProcessRealTime
),
mFilesDir(filesDir),
haveMusicFile(false),
exitMusicFeedThread(false),
android_fd(0)
{
LOGI("PlaybackEngine()");
LOGI("NDK LOG_LEVEL=%d", LOG_LEVEL);
// NDK LOG_LEVEL=3 (DEBUG)
@@ -62,9 +91,148 @@ PlaybackEngine::PlaybackEngine(std::string filesDir, int resid): mFilesDir(files
mPlayer = new MixingPlayer(samples);
int32_t res = mPlayer->startAudio();
LOGI("startAudio() = %d", res);
initRubberBand();
}
void PlaybackEngine::initRubberBand() {
// TODO: check mp3 actual sample rate, and adapt to 48000 Hz playback here
stretcher.setTimeRatio(1.0);
stretcher.setPitchScale(1.0);
stretcher.setDebugLevel(1); // 1: errors only. generally 0..4
// feed samples into 'stretcher' and read bogus output
// getPreferredStartPad() -> [ ... ] -> getStartDelay()
// write silence
// process() input: de-interleaved audio data with one float array per channel.
size_t num_pad = stretcher.getPreferredStartPad();
float* buf = (float*) malloc(num_pad*2*sizeof(float));
float* buf_ptr[] {buf, buf + num_pad};
memset(buf, 0, num_pad*2*sizeof(float));
stretcher.process(buf_ptr, num_pad, false);
free(buf);
//LOGI("start_pad = %d", num_pad);
// read bogus output
size_t start_delay = stretcher.getStartDelay();
float* out_buf = (float*) malloc(start_delay*2*sizeof(float));
float* out_buf_ptr[] {out_buf, out_buf + start_delay};
memset(out_buf, 0, start_delay*2*sizeof(float));
stretcher.retrieve(out_buf_ptr, start_delay);
//LOGI("start_delay = %d", start_delay);
// thread 1: oboe cb fetching via retrieve()
//mPlayer->setMusic();
// thread 2: polling for decoding more mp3 -> process() -- getSamplesRequired()
// setTimeRatio(), setPitchScale() -- always call them from thread 2
musicFeed = std::make_unique<std::thread>(&PlaybackEngine::musicFeedThread, this);
}
void PlaybackEngine::closeRubberBand() {
if(musicFeed) {
exitMusicFeedThread.store(true);
musicFeed->join();
musicFeed = nullptr;
}
closeMusicFile();
}
void PlaybackEngine::closeMusicFile() {
haveMusicFile.store(false);
musicFile = nullptr;
if(android_fd) {
close(android_fd);
android_fd = 0;
}
}
void PlaybackEngine::musicFeedThread() {
// refactor: rename to 'num_buf_samples'
size_t num_pad = 48000; // hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()?
size_t buf_stride = num_pad;
float* buf = (float*) malloc(num_pad*2*sizeof(float));
float* buf_ptr[] {buf, buf + num_pad};
memset(buf, 0, num_pad*2*sizeof(float));
unsigned char* cbuf = (unsigned char*) malloc(num_pad*2*sizeof(int16_t));
memset(cbuf, 0, num_pad*2*sizeof(int16_t));
size_t cbuf_size_bytes = num_pad*2*sizeof(int16_t);
int idebug = 0;
// thread 2: polling for decoding more mp3 -> process() -- getSamplesRequired()
while(!exitMusicFeedThread.load()) {
// do work ...
size_t num_samples = stretcher.getSamplesRequired();
// note: how much to sleep until output has played x samples...?
// can we just measure the wall time here, instead of calculating? -- that will be imprecise.
// how large is one buffer, and when do we feed it more data?
// (is it like double-buffering implemented in 'stretcher'?)
// can we just wait some bogus interval here, for a first version?
if (num_samples == 0) {
//LOGD("waiting for getSamplesRequired()");
std::this_thread::sleep_for(std::chrono::milliseconds(20));
continue;
}
if (num_samples > num_pad) {
LOGE("wanted %d samples but buf is only %d samples", num_samples, num_pad);
continue;
}
if (!haveMusicFile.load()) {
if(idebug++ < 10) {
LOGI("feed %d silence samples", num_samples);
// 1024, 512, 512
// 7 x 512
}
memset(buf, 0, num_samples*2*sizeof(float));
stretcher.process(buf_ptr, num_samples, false);
continue;
}
if(idebug++ < 10) {
LOGI("feed %d music samples", num_samples);
// feed 1024 music samples
// => stretcher is asking for 1024 = getSamplesRequired()
// ca. 21 ms
}
size_t done = 0; // bytes!
int err = mpg123_read(musicFile->handle, cbuf, cbuf_size_bytes, &done);
musicFile->remaining_samples -= done / sizeof(int16_t);
musicFile->offset = 0;
if (err != MPG123_OK && err != MPG123_DONE) {
// error!
LOGE("mpg123_read() err=%d done=%d", err, done);
// next iteration will play silence
closeMusicFile();
continue;
}
if(err == MPG123_DONE) {
// next iteration will play silence
closeMusicFile();
continue;
}
size_t num_decoded_samples = done / sizeof(int16_t) / 2; // 2 channels - TODO: actually use mp3 channels!! below, too. 2.
LOGI("num_decoded_samples = %d", num_decoded_samples);
// convert interleaved int16 to de-interleaved float [-1.0, 1.0] format
for(size_t i = 0; i < num_decoded_samples; i++) {
for(size_t j = 0; j < 2; j++) {
buf[i + buf_stride * j] = static_cast<float>(*(reinterpret_cast<int16_t*>(cbuf) + i*2 + j)) / 32768.0f;
}
}
LOGI("calling stretcher.process()");
stretcher.process(buf_ptr, num_decoded_samples, false);
}
}
PlaybackEngine::~PlaybackEngine() {
closeRubberBand();
LOGI("~PlaybackEngine()");
mPlayer->stopAudio();
delete mPlayer;
@@ -76,9 +244,57 @@ void PlaybackEngine::playBeat() {
}
void PlaybackEngine::playMusic(int fd) {
if(!mPlayer) return;
// TODO: fetch sampling rate from mp3 file, and use librubberband to correct for it
// MixingPlayer::kSampleRate (48000)
// mp3->rate
// feed samples to librubberband
// fetch resamples out of librubberband
//if(mPlayer) mPlayer->playMusic();
// TODO: fd is opened; dispose of fd when stopping or being discarded ...
LOGI("PlaybackEngine::playMusic(fd=%d)", fd);
close(fd); // for now, nothing is implemented. we just close it again.
//close(fd); // for now, nothing is implemented. we just close it again.
// we will use mp3file_open_fd() later.
android_fd = fd;
musicFile.reset(mp3file_open_fd(android_fd, 0));
haveMusicFile.store(true);
mPlayer->setMusic(std::make_shared<MusicProvider>(&stretcher));
}
MusicProvider::MusicProvider(RubberBand::RubberBandStretcher *stretcher) : stretcher(stretcher), idebug(0) {
// refactor: rename to 'num_buf_samples'
// TODO: for cache-friendliness, it would be better to have smaller 'num_buf_samples'
// hack! how much to actually reserve? is getPreferredStartPad() always < getSamplesRequired()?
//size_t buf_stride = num_pad;
buf = (float*) malloc(num_buf_samples*2*sizeof(float));
//float* buf_ptr[] {buf, buf + num_pad};
}
MusicProvider::~MusicProvider() {
free(buf);
}
void MusicProvider::onAudioReady(float *data, int32_t frames) {
if(idebug++ < 10) {
LOGI("onAudioReady() frames=%d", frames);
// frames=96 (48 kHz => 2 ms!!)
}
// 1. read from oboe into our temp de-interleaved buffer 'buf'
size_t num_frames = std::min((size_t) frames, num_buf_samples);
float* buf_ptr[] {buf, buf + num_buf_samples};
stretcher->retrieve(buf_ptr, num_frames);
// 2. convert to add samples to interleaved *data
for(size_t i = 0; i < num_frames; i++) {
for(size_t j = 0; j < 2; j++) {
float sample = data[i*2 + j];
sample += buf_ptr[j][i];
sample /= 2.0;
data[i*2 + j] = sample;
}
}
}

31
app/src/main/cpp/PlaybackEngine.h
View File

@@ -7,7 +7,28 @@
#include "StepListener.h"
#include "MixingPlayer.h"
#include "RubberBandStretcher.h"
#include "mp3file.h"
#include "AudioCallback.h"
#include <string>
#include <thread>
#include <memory>
#include <atomic>
/** Provides music through a regular callback to oboe. Called from separate oboe thread. */
class MusicProvider : public AudioCallbackProvider {
public:
explicit MusicProvider(RubberBand::RubberBandStretcher *stretcher);
~MusicProvider() override;
/** Called from separate oboe thread. */
void onAudioReady(float *data, int32_t frames) override;
private:
const size_t num_buf_samples = 48000;
RubberBand::RubberBandStretcher *stretcher;
float *buf;
int idebug;
};
class PlaybackEngine : public StepListener {
public:
@@ -17,8 +38,18 @@ public:
virtual void playBeat();
void playMusic(int fd);
private:
RubberBand::RubberBandStretcher stretcher;
MixingPlayer *mPlayer;
std::string mFilesDir;
std::unique_ptr<MP3File> musicFile;
std::atomic<bool> haveMusicFile;
std::unique_ptr<std::thread> musicFeed;
std::atomic<bool> exitMusicFeedThread;
int android_fd;
void initRubberBand();
void closeRubberBand();
void closeMusicFile();
void musicFeedThread();
};
#endif //LOCKSTEP_PLAYBACKENGINE_H

2
app/src/main/cpp/mp3file.cpp
View File

@@ -151,7 +151,7 @@ MP3File* mp3file_open_fd(int fd, int forceEncoding) {
mp3->offset = 0;
mp3->remaining_samples = (int) mp3->num_samples;
LOGV("channels: %d rate: %ld num_samples: %ld", mp3->channels, mp3->rate, mp3->num_samples);
LOGI("channels: %d rate: %ld num_samples: %ld", mp3->channels, mp3->rate, mp3->num_samples);
mp3->android_fd = fd;
return mp3;

4
app/src/main/java/at/lockstep/app/LstForegroundService.java
View File

@@ -87,7 +87,9 @@ public class LstForegroundService extends Service implements SensorEventListener
if (ACTION_START.equals(action)) {
String contentUri = intent.getStringExtra("content_uri");
try {
PlaybackEngine.playMusic(uriToFd(contentUri));
if(contentUri != null) {
PlaybackEngine.playMusic(uriToFd(contentUri));
}
} catch (IOException e) {
// TODO proper error handling
Toast.makeText(this, "Could not open music file contentUri", Toast.LENGTH_LONG).show();