feat: detect steps and play audio

TODO.md

@@ -1,3 +1,12 @@
+## TODO
+
+O> 16 KB paging for NDK libs
+* target SDK level 35
+
+* minimum amplitude for accelero
+* filter away bad SQI areas of the signal (do not detect steps if we have bad SQI)
+
+## Nice-To
 * myband PlaybackEngine.cpp has latency management and other audio performance related features.
   Check if the app can be improved (audio wise) by using that code instead.
 

app/build.gradle

@@ -6,6 +6,7 @@ plugins {
 android {
     namespace 'at.lockstep'
     compileSdk 34
+    ndkVersion '29.0.14206865'
 
     defaultConfig {
         applicationId "at.lockstep"
@@ -21,15 +22,19 @@ android {
         externalNativeBuild {
             cmake {
                 //path 'src/main/cpp/CMakeLists.txt'
-                cppFlags ''
+                //cppFlags ''
                 arguments "-DANDROID_STL=c++_shared"
 
                 //cppFlags "-std=c++14"
                 //arguments '-DANDROID_STL=c++_static'
 
+                //cppFlags "-Wl,-z,max-page-size=16384 -Wl,-z,common-page-size=4096"
+                // should be provided by default by newer NDK (NDK r29)
+
                 // armeabi and mips are deprecated in NDK r16 so we don't want to build for them
                 // TODO: android manifest filters to include only these hardware archs
-                abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86_64' //, 'x86', 'x86_64'
+                //abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86_64' //, 'x86', 'x86_64'
+                abiFilters 'armeabi-v7a', 'arm64-v8a' //, 'aarch64' // 'arm64-v8a' ???
             }
         }
     }

app/src/main/AndroidManifest.xml

@@ -2,6 +2,10 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     xmlns:tools="http://schemas.android.com/tools">
 
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
+    <uses-permission android:name="android.permission.WAKE_LOCK" />
+
     <application
         android:allowBackup="true"
         android:dataExtractionRules="@xml/data_extraction_rules"
@@ -23,6 +27,11 @@
                 <category android:name="android.intent.category.LAUNCHER" />
             </intent-filter>
         </activity>
+        <service
+            android:name="at.lockstep.app.LstForegroundService"
+            android:exported="false"
+            android:foregroundServiceType="mediaPlayback"
+            />
     </application>
 
 </manifest>

app/src/main/cpp/CMakeLists.txt

@@ -12,6 +12,8 @@ cmake_minimum_required(VERSION 3.22.1)
 # build script scope).
 project("lockstep-native")
 
+add_subdirectory(libpasada/pasada-lib)
+
 # Creates and names a library, sets it as either STATIC
 # or SHARED, and provides the relative paths to its source code.
 # You can define multiple libraries, and CMake builds them for you.
@@ -31,6 +33,8 @@ add_library(${CMAKE_PROJECT_NAME} SHARED
     PlaybackEngine.cpp
     mp3file.cpp
     jni_bridge.cpp
+    jni_stepdetector.cpp
+    StepDetector.cpp
 )
 
 find_package (oboe REQUIRED CONFIG)
@@ -45,11 +49,14 @@ set_target_properties(mpg123 PROPERTIES IMPORTED_LOCATION
         ${mpg123_DIR}/lib/${ANDROID_ABI}/libmpg123.so)
 include_directories(${mpg123_DIR}/lib/${ANDROID_ABI}/include)
 
+target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE libpasada/pasada-lib/include)
+
 # Specifies libraries CMake should link to your target library. You
 # can link libraries from various origins, such as libraries defined in this
 # build script, prebuilt third-party libraries, or Android system libraries.
 target_link_libraries(${CMAKE_PROJECT_NAME}
     # List libraries link to the target library
+    pasada
     oboe::oboe
     mpg123
     android

app/src/main/cpp/MixingPlayer.h

@@ -37,7 +37,7 @@ public:
         std::lock_guard<std::mutex> lock(mLock);
         oboe::AudioStreamBuilder builder;
         // The builder set methods can be chained for convenience.
-        Result result = builder.setSharingMode(oboe::SharingMode::Exclusive)
+        Result result = builder.setSharingMode(oboe::SharingMode::Shared)
                 ->setPerformanceMode(oboe::PerformanceMode::LowLatency)
                 ->setChannelCount(kChannelCount)
                 ->setSampleRate(kSampleRate)

app/src/main/cpp/PlaybackEngine.cpp

@@ -56,3 +56,7 @@ PlaybackEngine::~PlaybackEngine() {
     delete mPlayer;
     mPlayer = nullptr;
 }
+
+void PlaybackEngine::playBeat() {
+    if(mPlayer) mPlayer->setStartBeat();
+}

app/src/main/cpp/PlaybackEngine.h

@@ -5,13 +5,16 @@
 #ifndef LOCKSTEP_PLAYBACKENGINE_H
 #define LOCKSTEP_PLAYBACKENGINE_H
 
+#include "StepListener.h"
 #include "MixingPlayer.h"
 #include <string>
 
-class PlaybackEngine {
+class PlaybackEngine : public StepListener {
 public:
     PlaybackEngine(std::string filesDir, int resid);
     virtual ~PlaybackEngine();
+    /** Play a beat sound. */
+    virtual void playBeat();
 private:
     MixingPlayer *mPlayer;
     std::string mFilesDir;

app/src/main/cpp/StepDetector.cpp

@@ -0,0 +1,44 @@
+//
+// Created by david on 03.03.2026.
+//
+
+#include "StepDetector.h"
+
+// TODO: we are hardcoding filter coefficients for 60 Hz
+// TODO: this is tolerable for 50 Hz
+
+// TODO: check if we can do with floats instead of doubles
+// (check how much the [already bad] accuracy of filtering suffers)
+
+// TODO: in Java, check if delta timestamps effectively match FPS
+// TODO: FPS constant should be passed as argument to C++ (but we keep an FPS define to validate the coefficients)
+
+// Butterworth filter: order=5, fc=0.5, fs=60, btype='highpass'
+static std::vector hpf_taps_b {0.91875845, -4.59379227,  9.18758454, -9.18758454,  4.59379227, -0.91875845};
+static std::vector hpf_taps_a {1.        , -4.83056552,  9.33652742, -9.02545247,  4.36360803, -0.8441171};
+static size_t upslope_width = 4;
+const size_t len_refr = (size_t) (FPS / (MAX_BPM / 60));
+
+StepDetector::StepDetector(StepListener *listener) :
+    listener(listener),
+    f_highpass(hpf_taps_b, hpf_taps_a),
+    f_neg(1, 0, 0, std::vector {-1.0}),
+    f_ssf(upslope_width),
+    f_ssd(len_refr)
+{}
+
+#if (FPS != 60)
+#error "FPS must currently be 60, as highpass taps are pre-computed for that value"
+#endif
+
+void StepDetector::filter(std::vector<float> values) {
+    // TODO: later on, we should use a vector projection towards gravity
+    auto s0 = (double) values[1]; // take y-axis value for now
+    auto s1 = f_highpass.filter(s0);
+    auto s2 = f_neg.filter(s1);
+    auto s3 = f_ssf.filter(s2);
+    auto s4 = f_ssd.filter(s3);
+    if(s4 > 0.0 && listener != nullptr) {
+        listener->playBeat();
+    }
+}

app/src/main/cpp/StepDetector.h

@@ -0,0 +1,26 @@
+//
+// Created by david on 03.03.2026.
+//
+
+#ifndef LOCKSTEP_STEPDETECTOR_H
+#define LOCKSTEP_STEPDETECTOR_H
+
+#include "StepListener.h"
+#include "iir_filter.h"
+#include "ssf_filter.h"
+#include <vector>
+
+class StepDetector {
+protected:
+    StepListener *listener;
+    IirFilter f_highpass;
+    Filt f_neg;
+    SsfFilter f_ssf;
+    SsfStepDetector f_ssd;
+
+public:
+    StepDetector(StepListener *listener);
+    void filter(std::vector<float> values);
+};
+
+#endif //LOCKSTEP_STEPDETECTOR_H

app/src/main/cpp/StepListener.h

@@ -0,0 +1,14 @@
+//
+// Created by david on 03.03.2026.
+//
+
+#ifndef LOCKSTEP_STEPLISTENER_H
+#define LOCKSTEP_STEPLISTENER_H
+
+class StepListener {
+public:
+    virtual ~StepListener() {}
+    virtual void playBeat() = 0;
+};
+
+#endif //LOCKSTEP_STEPLISTENER_H

app/src/main/cpp/jni_stepdetector.cpp

@@ -0,0 +1,52 @@
+//
+// Created by david on 03.03.2026.
+//
+
+#include <jni.h>
+#include "StepDetector.h"
+#include <new>
+#include <vector>
+
+extern "C" {
+
+jint throwIllegalArgumentException(JNIEnv *env, const char *message)
+{
+    jclass exClass;
+    const char *className = "java/lang/IllegalArgumentException";
+    exClass = env->FindClass(className);
+    if (exClass == nullptr) {
+        return -1;
+    }
+    return env->ThrowNew(exClass, message);
+}
+
+JNIEXPORT jlong JNICALL
+Java_at_lockstep_filter_StepDetector_native_1create(
+        JNIEnv *env,
+        jclass /*unused*/, jlong engineHandle) {
+    auto *listener = reinterpret_cast<StepListener *>(engineHandle);
+    // We use std::nothrow so `new` returns a nullptr if the engine creation fails
+    auto *detector = new(std::nothrow) StepDetector(listener);
+    return reinterpret_cast<jlong>(detector);
+}
+
+JNIEXPORT void JNICALL
+Java_at_lockstep_filter_StepDetector_native_1delete(
+        JNIEnv *env,
+        jclass /*unused*/, jlong handle) {
+    delete reinterpret_cast<StepDetector *>(handle);
+}
+
+JNIEXPORT void JNICALL
+Java_at_lockstep_filter_StepDetector_native_1filter(
+        JNIEnv *env,
+        jclass /*unused*/, jlong handle, jlong timestamp, jfloatArray values) {
+    if(values == nullptr) throwIllegalArgumentException(env, "values == null");
+    float* nativeValues = (float *)env->GetFloatArrayElements(values, 0);
+    jsize length = env->GetArrayLength(values);
+    std::vector<float> vecValues(nativeValues, nativeValues + length);
+    auto *detector = reinterpret_cast<StepDetector *>(handle);
+    detector->filter(vecValues);
+}
+
+}

app/src/main/java/at/lockstep/app/LstForegroundService.java

@@ -0,0 +1,182 @@
+package at.lockstep.app;
+
+import android.app.Notification;
+import android.app.NotificationChannel;
+import android.app.NotificationManager;
+import android.app.Service;
+import android.content.Context;
+import android.content.Intent;
+import android.hardware.Sensor;
+import android.hardware.SensorEvent;
+import android.hardware.SensorEventListener;
+import android.hardware.SensorManager;
+import android.os.Build;
+import android.os.IBinder;
+import android.os.PowerManager;
+
+import androidx.annotation.Nullable;
+import androidx.core.app.NotificationCompat;
+
+import at.lockstep.filter.StepDetector;
+import at.lockstep.pb.PlaybackEngine;
+import at.lockstep.R;
+
+public class LstForegroundService extends Service implements SensorEventListener {
+
+    private static final String CHANNEL_ID = "sensor_collection_channel";
+    private static final int NOTIFICATION_ID = 1001;
+
+    public static final String ACTION_START = "at.lockstep.action.START";
+    public static final String ACTION_STOP = "at.lockstep.action.STOP";
+
+    private SensorManager sensorManager;
+    private Sensor accelerometer;
+    private PowerManager.WakeLock wakeLock;
+    private boolean isCollecting = false;
+
+    private StepDetector stepDetector;
+
+    public static Intent startIntent(Context context) {
+        Intent intent = new Intent(context, LstForegroundService.class);
+        intent.setAction(ACTION_START);
+        return intent;
+    }
+
+    public static Intent stopIntent(Context context) {
+        Intent intent = new Intent(context, LstForegroundService.class);
+        intent.setAction(ACTION_STOP);
+        return intent;
+    }
+
+    @Override
+    public void onCreate() {
+        super.onCreate();
+
+        int resid = R.raw.track_beat;
+        PlaybackEngine.create(this, resid);
+        stepDetector = new StepDetector(PlaybackEngine.getEngineHandle());
+
+        sensorManager = (SensorManager) getSystemService(Context.SENSOR_SERVICE);
+        if (sensorManager != null) {
+            accelerometer = sensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER);
+        }
+
+        createNotificationChannel();
+
+        PowerManager powerManager = (PowerManager) getSystemService(Context.POWER_SERVICE);
+        if (powerManager != null) {
+            wakeLock = powerManager.newWakeLock(
+                    PowerManager.PARTIAL_WAKE_LOCK,
+                    getPackageName() + ":SensorCollectionWakeLock"
+            );
+            wakeLock.setReferenceCounted(false);
+        }
+    }
+
+    @Override
+    public int onStartCommand(Intent intent, int flags, int startId) {
+        if (intent != null) {
+            String action = intent.getAction();
+            if (ACTION_START.equals(action)) {
+                startCollection();
+            } else if (ACTION_STOP.equals(action)) {
+                stopCollectionAndSelf();
+            }
+        }
+        return START_STICKY;
+    }
+
+    private void startCollection() {
+        if (isCollecting) {
+            return;
+        }
+
+        startForeground(NOTIFICATION_ID, buildNotification("Collecting sensor data"));
+
+        if (wakeLock != null && !wakeLock.isHeld()) {
+            // TODO: provide a timeout reasonable for a run
+            wakeLock.acquire(2*60*60*1000L /*2 hours*/);
+        }
+
+        if (accelerometer != null && sensorManager != null) {
+            sensorManager.registerListener(
+                    this,
+                    accelerometer,
+                    SensorManager.SENSOR_DELAY_GAME
+            );
+            isCollecting = true;
+        } else {
+            stopCollectionAndSelf();
+        }
+    }
+
+    private void stopCollectionAndSelf() {
+        if (isCollecting && sensorManager != null) {
+            sensorManager.unregisterListener(this);
+            isCollecting = false;
+        }
+
+        if (wakeLock != null && wakeLock.isHeld()) {
+            wakeLock.release();
+        }
+
+        stopForeground(STOP_FOREGROUND_REMOVE);
+        stopSelf();
+    }
+
+    @Override
+    public void onDestroy() {
+        if (sensorManager != null) {
+            sensorManager.unregisterListener(this);
+        }
+
+        if (wakeLock != null && wakeLock.isHeld()) {
+            wakeLock.release();
+        }
+
+        // TODO: check threading to see if these run in separate threads - if so, deleting PlaybackEngine will leave a dangling pointer in StepDetector.
+        // 2026-03-04 01:26:11.741 12507-12507 libc                    at.lockstep                          A  Fatal signal 11 (SIGSEGV), code 2 (SEGV_ACCERR), fault addr 0xb4000071d3a79000 in tid 12507 (at.lockstep), pid 12507 (at.lockstep)
+        stepDetector.close();
+        PlaybackEngine.delete();
+
+        super.onDestroy();
+    }
+
+    @Nullable
+    @Override
+    public IBinder onBind(Intent intent) {
+        return null;
+    }
+
+    @Override
+    public void onSensorChanged(SensorEvent event) {
+        stepDetector.filter(event.timestamp, event.values);
+    }
+
+    @Override
+    public void onAccuracyChanged(Sensor sensor, int accuracy) {}
+
+    private Notification buildNotification(String contentText) {
+        return new NotificationCompat.Builder(this, CHANNEL_ID)
+                .setContentTitle("Lockstep is reading your pace.")
+                .setContentText(contentText)
+                .setSmallIcon(android.R.drawable.ic_menu_compass)
+                .setOngoing(true)
+                .build();
+    }
+
+    private void createNotificationChannel() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            NotificationChannel channel = new NotificationChannel(
+                    CHANNEL_ID,
+                    "Lockstep",
+                    NotificationManager.IMPORTANCE_LOW
+            );
+
+            NotificationManager nm = getSystemService(NotificationManager.class);
+            if (nm != null) {
+                nm.createNotificationChannel(channel);
+            }
+        }
+    }
+}

app/src/main/java/at/lockstep/app/MainActivity.java

@@ -1,24 +1,42 @@
 package at.lockstep.app;
 
 import android.app.Activity;
+import android.os.Bundle;
+import android.widget.Button;
 
+import androidx.core.content.ContextCompat;
+
+import at.lockstep.R;
 import at.lockstep.pb.PlaybackEngine;
 
 public class MainActivity extends Activity {
-    /*
+    private Button btnStart;
-     * Creating engine in onResume() and destroying in onPause() so the stream retains exclusive
+    private Button btnStop;
-     * mode only while in focus. This allows other apps to reclaim exclusive stream mode.
-     */
-    @Override
-    protected void onResume() {
-        super.onResume();
-        int resid = at.lockstep.R.raw.track_beat;
-        PlaybackEngine.create(this, resid); // note: called twice (is permission request causing Activity to go out of focus?)
-    }
 
     @Override
-    protected void onPause() {
+    protected void onCreate(Bundle savedInstanceState) {
-        PlaybackEngine.delete();
+        super.onCreate(savedInstanceState);
-        super.onPause();
+        setContentView(R.layout.activity_main);
+
+        btnStart = findViewById(R.id.btnStart);
+        btnStop = findViewById(R.id.btnStop);
+
+        btnStart.setOnClickListener(v ->
+                ContextCompat.startForegroundService(
+                        MainActivity.this,
+                        LstForegroundService.startIntent(MainActivity.this)
+                )
+        );
+
+        //if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
+        //        startForeground(SERVICE_ID, notification)
+        //} else {
+        //        startForeground(SERVICE_ID, notification,
+        //FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK)
+        //}
+
+        btnStop.setOnClickListener(v ->
+                startService(LstForegroundService.stopIntent(MainActivity.this))
+        );
     }
 }

app/src/main/java/at/lockstep/filter/StepDetector.java

@@ -0,0 +1,21 @@
+package at.lockstep.filter;
+
+import at.lockstep.pb.PlaybackEngine;
+
+public class StepDetector {
+    long handle;
+
+    public StepDetector(long engineHandle) {
+        handle = native_create(engineHandle);
+    }
+    public void close() {
+        native_delete(handle);
+    }
+    public void filter(long timestamp, float[] values) {
+        native_filter(handle, timestamp, values);
+    }
+
+    private static native long native_create(long engineHandle);
+    private static native void native_delete(long sdHandle);
+    private static native void native_filter(long sdHandle, long timestamp, float[] values);
+}

app/src/main/java/at/lockstep/pb/PlaybackEngine.java

@@ -67,6 +67,9 @@ public class PlaybackEngine {
         }
         mEngineHandle = 0;
     }
+    public static long getEngineHandle() {
+        return mEngineHandle;
+    }
 
     private static native long native_createEngine(String filesDir, int resid);
     private static native void native_deleteEngine(long engineHandle);

app/src/main/res/layout/activity_main.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:gravity="center"
+    android:orientation="vertical"
+    android:padding="24dp">
+
+    <Button
+        android:id="@+id/btnStart"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="Start collection" />
+
+    <Button
+        android:id="@+id/btnStop"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_marginTop="16dp"
+        android:text="Stop collection" />
+</LinearLayout>