Files
hsh-prevent/app/src/main/java/net/heartshield/prevent/MeasureActivity.java

690 lines
27 KiB
Java
Raw Normal View History

2026-03-02 06:19:34 +01:00
package net.heartshield.prevent;
import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.content.res.TypedArray;
import android.os.Bundle;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Vibrator;
import android.util.Log;
import android.view.TextureView;
import android.widget.ImageView;
import android.widget.TextView;
import android.widget.Toast;
import com.github.mikephil.charting.charts.LineChart;
import net.heartshield.control.Device;
import net.heartshield.control.IMeasurementController;
import net.heartshield.control.MeasurementController;
import net.heartshield.data.AppInfo;
import net.heartshield.data.IMeasurementDataManager;
import net.heartshield.data.Measurement;
import net.heartshield.data.MeasurementDataManager;
import net.heartshield.data.UserMeta;
import net.heartshield.filter.PlotFilter;
import net.heartshield.sensors.IntensityDetector;
import net.heartshield.sensors.SensorFactory;
import net.heartshield.signal.DVec;
import net.heartshield.signal.RunningQuality;
import net.heartshield.signal.Signal;
import net.heartshield.ui.LineChartWrapper;
import static net.heartshield.prevent.MeasureActivity.LogMessageType.ERROR;
import static net.heartshield.prevent.MeasureActivity.LogMessageType.MEASUREMENT_FAILURE;
/**
* Measurement screen with countdown timer and running wave shape.
*
* @author David Madl (git@abanbytes.eu)
* @date 2016-04-01
*/
public class MeasureActivity extends Activity {
private static final String TAG = "MeasureActivity";
private UserMeta mUser;
private IMeasurementController.MeasurementMode mMode;
private IMeasurementController mController;
private IMeasurementDataManager mDataManager;
private AppInfo mAppInfo;
private HandlerThread mBackgroundThread;
private Handler mBackgroundHandler;
private LineChartWrapper mChart;
PlotFilter mPlotFilter;
private static final double FPS = 30.0;
/** we require a lock (including SQI lock) by this amount of time, and fail the measurement otherwise. this should allow plenty of time to settle all swings and get solid beats detected. */
//final double REQUIRE_LOCK_BY_TIME = 30.0;
private static final double TOTAL_TIME_RISK = 75.0;
public static final double FRONT_TIME_ENDOTHELIAL_FUNCTION = 75.0;
private static final double TOTAL_TIME_ENDOTHELIAL_FUNCTION = FRONT_TIME_ENDOTHELIAL_FUNCTION + 300.0 + 75.0;
public static final double TOTAL_TIME_VITAL_CHECK = 10.0;
public static final double FRONT_TIME_VITAL_CHECK = 5.0; // until when it is OK to restart measurement on large swings (e.g. when placing the phone)
double mTotalTime;
//final double TOTAL_TIME = 15.0;
final boolean TEST_MODE = false; // if true, do not save Measurement on server
private TextView mCountdownText;
private TextView mHeartRateText;
private ImageView mProgressCircle;
private ImageView mSignalIndicator;
private TextureView mCameraPreview = null;
private TypedArray mProgSequence;
private TypedArray mTxSequence;
private int mNumNoisy = 0;
private static final int MAX_NUM_NOISY = 15; // + give a few for Signal.beatdetect() errors.
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_measure);
//mCameraPreview = (TextureView) findViewById(R.id.cameraPreview);
mCountdownText = (TextView) findViewById(R.id.countdown);
mHeartRateText = (TextView) findViewById(R.id.heartRate);
mProgressCircle = (ImageView) findViewById(R.id.progressCircle);
mSignalIndicator = (ImageView) findViewById(R.id.signalIndicator);
mProgSequence = getResources().obtainTypedArray(R.array.progress_circle_sequence);
mTxSequence = getResources().obtainTypedArray(R.array.signal_transmit_sequence);
mHeartRateText.setText("- ");
mSignalIndicator.setImageResource(0);
mProgressCircle.invalidate();
mController = new MeasurementController(getMainLooper(), new Device(this), new SensorFactory(this), mCameraPreview);
mController.setListener(mListener);
// server does not save then. maybe we do want to save... maybe create another test flag, for not saving?
//if(AppInfo.getInstance(this).devSettings.skipDialogs)
// mController.setUnderTest(true);
mChart = new LineChartWrapper((LineChart) findViewById(R.id.chart));
final int PLOT_SAMPLES = (int) (FPS * 3.0); // see LineChartWrapper.WINDOW_SIZE_SECS
// TODO(david): PlotFilter should be more robust, not just TOTAL_TIME
mPlotFilter = new PlotFilter(FPS, (int) ((mTotalTime + 10.0) * FPS + 1), PLOT_SAMPLES);
/*
* when integrating RunningQuality into MeasurementController, keep in mind the necessity
* of a separate background worker thread.
*/
mQuality = new RunningQuality(FPS);
mQuality.setListener(new RunningQuality.BeatListener() {
private int mBadBeatStartIndex = 0;
@Override
public void onLocked(int startIdx) {
Log.i(TAG, "onLocked() at startIdx=" + startIdx);
mNumNoisy = 0;
mBadBeatStartIndex = 0;
double qualityLockTime = mQuality.getTimeAt(startIdx);
mController.setQualityLockTime(qualityLockTime);
}
@Override
public void onBeat(int startIdx, boolean goodBeat, double posCorrCoeff) {
//double time = mQuality.getTimeAt(startIdx);
Log.i(TAG, "onBeat() SQI: posCorrCoeff=" + posCorrCoeff);
if(!goodBeat) {
mNumNoisy++;
double t = mQuality.getTimeAt(startIdx);
Log.i(TAG, "onBeat() NOISY BEAT");
Log.i(TAG, "onBeat() NOISY BEAT");
Log.i(TAG, "onBeat() NOISY BEAT <<<<<<<< noisy beat #" + mNumNoisy);
Log.i(TAG, "onBeat() NOISY BEAT");
Log.i(TAG, "onBeat() NOISY BEAT");
}
/*
// ignored for doctors
if(mNumNoisy > MAX_NUM_NOISY) {
try {
mController.stop();
} catch (IMeasurementController.FramesDroppedException e) {
e.printStackTrace();
startErrorActivity(getString(R.string.frames_dropped));
return;
}
onMeasurementFailure(mController.getMeasurement());
}
*/
}
});
mDataManager = MeasurementDataManager.getInstance(getApplicationContext());
mAppInfo = AppInfo.getInstance(this);
}
private class SignalAnimator extends Thread {
private boolean running = true;
private int i;
SignalAnimator() {
start();
}
synchronized private boolean isRunning() { return running; }
@Override
public void run() {
i = 0;
while(isRunning()) {
try {
Thread.sleep(250);
} catch (InterruptedException e) {
e.printStackTrace();
break;
}
//mSignalIndicator.setImageResource();
final int ii = i;
if(++i >= mTxSequence.length())
i = 0;
runOnUiThread(new Runnable() {
@Override
public void run() {
mSignalIndicator.setImageResource(mTxSequence.getResourceId(ii, -1));
mSignalIndicator.invalidate();
}
});
}
}
synchronized void cancel() {
running = false;
interrupt();
try {
join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
private SignalAnimator mAnimator;
private void startAnimation() {
if(mAnimator != null)
return; // already animating
mAnimator = new SignalAnimator();
}
private void stopAnimation() {
if(mAnimator != null) {
mAnimator.cancel();
mAnimator = null;
}
}
@Override
protected void onResume() {
super.onResume();
mUser = (UserMeta) getIntent().getSerializableExtra("meta");
mMode = (IMeasurementController.MeasurementMode) getIntent().getSerializableExtra("mode");
Log.i(TAG, "got age=" + mUser.age + " gender=" + mUser.gender + " mode=" + mMode);
if(mMode == null) {
// to do: e.g. 'Repeat measurement' button in ResultActivity
Log.w(TAG, "should migrate all calling Activities to include mode");
mMode = IMeasurementController.MeasurementMode.RISK;
}
switch(mMode) {
default: // fall through
case RISK:
mTotalTime = TOTAL_TIME_RISK;
break;
case ENDOTHELIAL_FUNCTION:
mTotalTime = TOTAL_TIME_ENDOTHELIAL_FUNCTION;
break;
case VITAL_CHECK:
mTotalTime = TOTAL_TIME_VITAL_CHECK;
break;
}
mController.setMode(mMode);
mController.setUnderTest(mDataManager.isUnderTest());
mPlotFilter.clear();
mQuality.clear();
mController.start();
if(mDataManager.getNumFailedMeasurements() > 0)
mQuality.setBeatCorrThr2(RunningQuality.BEAT_CORR_THR_2_LOWER);
else
mQuality.setBeatCorrThr2(RunningQuality.BEAT_CORR_THR_2);
mCountdownText.setText(Integer.toString((int) mTotalTime));
mProgressCircle.setImageResource(mProgSequence.getResourceId(0, -1));
mProgressCircle.invalidate();
mSignalIndicator.setImageResource(0);
mProgressCircle.invalidate();
startBackgroundThread();
mDataManager.startBackgroundThread();
}
private void onMeasurementFinished(Measurement measurement) {
updateMeasurement(measurement);
startAnimation();
// if dummy microphone, pretend we're already synced
if(measurement.series.getAudio().length == 0)
measurement.series.audioSynced = true;
mDataManager.create(measurement, new IMeasurementDataManager.MeasurementResultListener() {
@Override
public void onResult(Measurement measurement) {
//assert(measurement.result != null);
// successful result from server
Intent intent = new Intent(MeasureActivity.this, ResultActivity.class);
intent.putExtra("measurementId", measurement.getId());
startActivity(intent);
stopAnimation();
}
@Override
public void onError(Measurement measurement, IMeasurementDataManager.ErrorCode code, String message) {
onCreateError(measurement, code, message);
}
});
}
private void onCreateError(Measurement measurement, IMeasurementDataManager.ErrorCode code, String message) {
// some error saving the stuff
Toast.makeText(MeasureActivity.this, "Failed to get result: " + code + " " + message, Toast.LENGTH_LONG).show();
Log.e(TAG, "Failed to get result: " + code + " " + message);
stopAnimation();
int failureReason;
// note: UNCLASSIFIED is a regular, protocol-level response that is not an error.
// hence it is not handled here, but displayed in ResultActivity in a special way
switch(code) {
case CONNECTION_ERROR:
failureReason = R.drawable.signal_none;
break;
case HTTP_ERROR:
case LOCAL_ERROR:
case UNKNOWN_ERROR:
case PROTOCOL_ERROR:
failureReason = R.drawable.signal_fail;
break;
default:
failureReason = R.drawable.signal_fail;
}
mSignalIndicator.setImageResource(failureReason);
mSignalIndicator.invalidate();
}
enum LogMessageType {
MEASUREMENT_FAILURE,
ERROR
}
private void onMeasurementFailure(Measurement measurement) {
updateMeasurement(measurement);
int numFailedBefore = mDataManager.getNumFailedMeasurements();
mDataManager.logMessage(measurement.appId, MEASUREMENT_FAILURE.toString(), "measurement failure. numFailedBefore=" + numFailedBefore);
mDataManager.addFailedMeasurement();
startAnimation();
mDataManager.logFailure(measurement, new IMeasurementDataManager.MeasurementResultListener() {
@Override
public void onResult(Measurement measurement) {
// show result from server (UNCLASSIFIED)
Intent intent = new Intent(MeasureActivity.this, ResultActivity.class);
intent.putExtra("measurementId", measurement.getId());
startActivity(intent);
stopAnimation();
}
@Override
public void onError(Measurement measurement, IMeasurementDataManager.ErrorCode code, String message) {
onCreateError(measurement, code, message);
}
});
}
private void updateMeasurement(Measurement measurement) {
measurement.user = mUser;
measurement.app = mAppInfo.getAppMeta();
measurement.meta.mode = mMode;
measurement.meta.test = TEST_MODE;
// if dummy microphone, pretend we're already synced
if(measurement.series.getAudio().length == 0)
measurement.series.audioSynced = true;
}
@Override
protected void onPause() {
super.onPause();
try {
mController.stop();
} catch (IMeasurementController.FramesDroppedException e) {
// ignored, since result is discarded here anyways
e.printStackTrace();
}
stopAnimation();
stopBackgroundThread();
mDataManager.stopBackgroundThread();
}
/**
* Two layers of signal quality checks:
*
* 1. finger presence: (redness, uniformity, limit brightness-swings)
* * restarts if you take off your finger
* vibrates on restarts
* * 30s timer limit if never locked for at least 10s
* TODO: BUG: if never locked, timeSinceLock keeps increasing.
*
* 2. pulse beat quality: (beat detection and cross-correlation of beats)
* * locks if 2+ similar beats are found
* * only shows BPM if the quality is locked
*/
private RunningQuality mQuality;
/** filter and plot the red channel of the camera */
private IMeasurementController.Listener mListener = new IMeasurementController.Listener() {
private int mI;
private IMeasurementController.State mPrevState = IMeasurementController.State.STOPPED;
private double mTimeSinceLock = 0.0;
private boolean mEverLocked = false;
private void clear() {
mI = 0;
mPrevSigTs = 0.0;
mTimeSinceLock = 0.0;
efBuzzed1 = false;
efBuzzed2 = false;
}
@Override
public void onState(IMeasurementController.State newState) {
if(newState == IMeasurementController.State.STOPPED || newState == IMeasurementController.State.FAILED)
clear();
if(newState == IMeasurementController.State.FAILED) {
if(mController.getFailureReason().equals(IMeasurementController.FailureReason.CAMERA_FRAMES_DROPPED)) {
startErrorActivity(getString(R.string.frames_dropped));
} else {
startErrorActivity("Error: Measurement failed: " + mController.getFailureReason().toString());
}
}
if(newState == IMeasurementController.State.SAMPLING) {
if(!mEverLocked) {
// short vibrate at first lock
Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE);
v.vibrate(200);
}
mEverLocked = true;
}
if(mPrevState == IMeasurementController.State.SAMPLING && newState == IMeasurementController.State.WAITING) {
// avoid message when user puts their finger on (bouncing camera lock)
if(mTimeSinceLock > 5.0) {
Toast.makeText(MeasureActivity.this, "Measurement restarted", Toast.LENGTH_SHORT).show();
}
mQuality.clear();
// note: both SQI and swing-level restarts go through here
// vibrate on restart
Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE);
v.vibrate(200);
}
mPrevState = newState;
}
private double mPrevSigTs = 0.0;
/** beatdetect() for SQI and BPM will run on the signal starting from (cameraLockTime + SQI_SETTLING_TIME) */
private static final double SQI_SETTLING_TIME = 3.0;
private boolean efBuzzed1 = false;
private boolean efBuzzed2 = false;
@Override
public void onFrame(final double timeSinceStart, final double timeSinceLock, double[] rgb, IntensityDetector.ImageSummary summary) {
final int RED_CHANNEL = 0;
final double entry = rgb[RED_CHANNEL];
Log.i(TAG, "ON_FRAME " + timeSinceStart + " " + timeSinceLock + " " + rgb[0] + " " + rgb[1] + " " + rgb[2]);
mTimeSinceLock = timeSinceLock;
if(++mI % 30 == 0) {
mCountdownText.setText(Integer.toString(((int) (mTotalTime - timeSinceLock))));
int progSeqId = (int) (mProgSequence.length() * timeSinceLock / mTotalTime);
progSeqId = Math.min(progSeqId, mProgSequence.length()-1);
mProgressCircle.setImageResource(mProgSequence.getResourceId(progSeqId, -1));
mProgressCircle.invalidate();
// when pressing Back, it happens that mBackgroundHandler is null here - onPause() called before running this callback Runnable
// we could check if Activity is paused before dispatching in MeasurementController, but then we'd need to hand the Activity to it.
if(mBackgroundHandler != null) {
if(mQuality.isRunning()) {
mBackgroundHandler.post(new Runnable() {
@Override
public void run() {
// takes some time, at longer signals (since it does full beat detection)
// run it in the background to avoid UI lag of the graph plotting
updateHeartRate(mController.getLockTime() + SQI_SETTLING_TIME);
}
});
}
//
// running signal quality
//
if(mPrevState == IMeasurementController.State.SAMPLING && timeSinceLock > SQI_SETTLING_TIME) {
mBackgroundHandler.post(new Runnable() {
@Override
public void run() {
// use signal since lock, otherwise the amplitude in Signal.beatdetect() takes too long to settle
if(mPrevSigTs == 0.0)
mPrevSigTs = mController.getLockTime() + SQI_SETTLING_TIME;
// must use detrended data, since the DC would mean the correlations will look too good to be true.
PlotFilter.Series ser = getSeriesSinceLock(mPrevSigTs);
if(ser.t.length > 0) {
mQuality.append(ser.t[0], ser.x);
Log.i(TAG, "mQuality.append() of pts.length=" + ser.x.length);
mPrevSigTs = ser.t[ser.t.length - 1];
}
}
});
}
}
}
// note: if the camera was never locked (unlikely), `timeSinceLock` reports same as `timeSinceStart`
//boolean lockOverdue = timeSinceLock > REQUIRE_LOCK_BY_TIME; // camera lock <--- long time ---> ?
boolean lockOverdue = false; // disabled for doctors - always finish the measurement.
if(lockOverdue && !mQuality.isRunning()) {
try {
mController.stop();
} catch (IMeasurementController.FramesDroppedException e) {
e.printStackTrace();
startErrorActivity(getString(R.string.frames_dropped));
return;
}
onMeasurementFailure(mController.getMeasurement());
return;
}
// bad: since more than 30 secs, there was no lock of over 10 sec duration?
boolean finished = timeSinceLock > mTotalTime;
boolean notLockedTrouble = (timeSinceStart > 30.0 && timeSinceLock < 10.0); // && (mMode.equals(IMeasurementController.MeasurementMode.RISK) || (mMode.equals(IMeasurementController.MeasurementMode.ENDOTHELIAL_FUNCTION) && timeSinceStart < TOTAL_TIME_RISK))
if(mMode.equals(IMeasurementController.MeasurementMode.ENDOTHELIAL_FUNCTION) && timeSinceLock > TOTAL_TIME_RISK && !efBuzzed1) {
// buzz 1: inflate cuff (provoke arm ischemia)
Toast.makeText(MeasureActivity.this, "Inflate cuff now", Toast.LENGTH_LONG).show();
Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE);
v.vibrate(200);
efBuzzed1 = true;
} else if(mMode.equals(IMeasurementController.MeasurementMode.ENDOTHELIAL_FUNCTION) && timeSinceLock > (TOTAL_TIME_ENDOTHELIAL_FUNCTION - TOTAL_TIME_RISK) && !efBuzzed2) {
// buzz 2: deflate cuff (remove arm ischemia)
Toast.makeText(MeasureActivity.this, "Deflate cuff now", Toast.LENGTH_LONG).show();
Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE);
v.vibrate(200);
efBuzzed2 = true;
}
if(finished || notLockedTrouble) {
// ensure we only call stop() once
if(mController.getState() == IMeasurementController.State.STOPPED)
return;
try {
Log.i(TAG, "since more than 30 secs, there was no lock of over 10 sec duration. timeSinceLock=" + timeSinceLock + " timeSinceStart=" + timeSinceStart);
mController.stop();
if(mQuality.isRunning() && mNumNoisy <= MAX_NUM_NOISY) {
// what if never locked properly? wrong API endpoint?
onMeasurementFinished(mController.getMeasurement());
} else {
// long vibrate on fail
Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE);
v.vibrate(new long[]{400, 400, 400, 400}, -1);
// codepath used for doctors
onMeasurementFailure(mController.getMeasurement());
}
} catch (IMeasurementController.FramesDroppedException e) {
e.printStackTrace();
startErrorActivity(getString(R.string.frames_dropped));
}
return;
}
final Runnable runnable = new Runnable() {
@Override
public void run() {
mPlotFilter.addUneven(timeSinceStart, entry);
PlotFilter.Series slice = mPlotFilter.getPlotSeries();
mChart.replaceData(slice.t, slice.x);
}
};
runOnUiThread(runnable);
}
};
private void startOtherActivity(Class<?> activityClass) {
Intent intent = new Intent(MeasureActivity.this, activityClass);
intent.putExtra("meta", mUser);
startActivity(intent);
stopAnimation();
}
private void startErrorActivity(String message) {
mDataManager.logMessage(mAppInfo.getAppMeta().appId, ERROR.toString(), message);
Intent intent = new Intent(MeasureActivity.this, ErrorActivity.class);
intent.putExtra("meta", mUser);
intent.putExtra("message", message);
startActivity(intent);
stopAnimation();
}
private PlotFilter.Series getSeriesSinceLock(double lockTime) {
int len = mPlotFilter.size();
PlotFilter.Series series = mPlotFilter.getRecentSeries(len);
int istart = 0;
for(; istart < series.t.length; istart++)
if(series.t[istart] > lockTime)
break;
double[] x = DVec.get(series.x, istart, series.x.length);
double[] t = DVec.get(series.t, istart, series.t.length);
return new PlotFilter.Series(t, x);
}
private void updateHeartRate(double sinceTime) {
int len = mPlotFilter.size();
// ignore the initial, large swing
PlotFilter.Series ser = getSeriesSinceLock(sinceTime);
double[] pts = ser.x;
// we should make sure that detrended amplitude swing is much smaller (< e.g. 16, see ipynb exploration) in recent stuff
Log.i(TAG, "updateHeartRate() len=" + len + " pointsSinceLock.length=" + pts.length);
final Double bpm = Signal.calculateBpm(FPS, pts);
if(pts.length >= 60) {
double[] smoothd = Signal.running_mean(Signal.running_mean(pts, 6), 7);
double[] slice = DVec.get(smoothd, smoothd.length - 30, smoothd.length);
Log.i(TAG, "swing amplitude: " + (DVec.max(slice) - DVec.min(slice)));
}
runOnUiThread(new Runnable() {
@Override
public void run() {
if(bpm != null)
mHeartRateText.setText(Integer.toString(bpm.intValue()));
else
mHeartRateText.setText("- ");
}
});
}
private void startBackgroundThread() {
mBackgroundThread = new HandlerThread("MeasureBackground");
mBackgroundThread.start();
mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
}
private void stopBackgroundThread() {
if(mBackgroundThread != null) {
mBackgroundThread.quitSafely();
try {
mBackgroundThread.join();
mBackgroundThread = null;
mBackgroundHandler = null;
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}