package net.heartshield.prevent; import android.app.Activity; import android.content.Context; import android.content.Intent; import android.content.res.TypedArray; import android.os.Bundle; import android.os.Handler; import android.os.HandlerThread; import android.os.Vibrator; import android.util.Log; import android.view.TextureView; import android.widget.ImageView; import android.widget.TextView; import android.widget.Toast; import com.github.mikephil.charting.charts.LineChart; import net.heartshield.control.Device; import net.heartshield.control.IMeasurementController; import net.heartshield.control.MeasurementController; import net.heartshield.data.AppInfo; import net.heartshield.data.IMeasurementDataManager; import net.heartshield.data.Measurement; import net.heartshield.data.MeasurementDataManager; import net.heartshield.data.UserMeta; import net.heartshield.filter.PlotFilter; import net.heartshield.sensors.IntensityDetector; import net.heartshield.sensors.SensorFactory; import net.heartshield.signal.DVec; import net.heartshield.signal.RunningQuality; import net.heartshield.signal.Signal; import net.heartshield.ui.LineChartWrapper; import static net.heartshield.prevent.MeasureActivity.LogMessageType.ERROR; import static net.heartshield.prevent.MeasureActivity.LogMessageType.MEASUREMENT_FAILURE; /** * Measurement screen with countdown timer and running wave shape. * * @author David Madl (git@abanbytes.eu) * @date 2016-04-01 */ public class MeasureActivity extends Activity { private static final String TAG = "MeasureActivity"; private UserMeta mUser; private IMeasurementController.MeasurementMode mMode; private IMeasurementController mController; private IMeasurementDataManager mDataManager; private AppInfo mAppInfo; private HandlerThread mBackgroundThread; private Handler mBackgroundHandler; private LineChartWrapper mChart; PlotFilter mPlotFilter; private static final double FPS = 30.0; /** we require a lock (including SQI lock) by this amount of time, and fail the measurement otherwise. this should allow plenty of time to settle all swings and get solid beats detected. */ //final double REQUIRE_LOCK_BY_TIME = 30.0; private static final double TOTAL_TIME_RISK = 75.0; public static final double FRONT_TIME_ENDOTHELIAL_FUNCTION = 75.0; private static final double TOTAL_TIME_ENDOTHELIAL_FUNCTION = FRONT_TIME_ENDOTHELIAL_FUNCTION + 300.0 + 75.0; public static final double TOTAL_TIME_VITAL_CHECK = 10.0; public static final double FRONT_TIME_VITAL_CHECK = 5.0; // until when it is OK to restart measurement on large swings (e.g. when placing the phone) double mTotalTime; //final double TOTAL_TIME = 15.0; final boolean TEST_MODE = false; // if true, do not save Measurement on server private TextView mCountdownText; private TextView mHeartRateText; private ImageView mProgressCircle; private ImageView mSignalIndicator; private TextureView mCameraPreview = null; private TypedArray mProgSequence; private TypedArray mTxSequence; private int mNumNoisy = 0; private static final int MAX_NUM_NOISY = 15; // + give a few for Signal.beatdetect() errors. @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_measure); //mCameraPreview = (TextureView) findViewById(R.id.cameraPreview); mCountdownText = (TextView) findViewById(R.id.countdown); mHeartRateText = (TextView) findViewById(R.id.heartRate); mProgressCircle = (ImageView) findViewById(R.id.progressCircle); mSignalIndicator = (ImageView) findViewById(R.id.signalIndicator); mProgSequence = getResources().obtainTypedArray(R.array.progress_circle_sequence); mTxSequence = getResources().obtainTypedArray(R.array.signal_transmit_sequence); mHeartRateText.setText("- "); mSignalIndicator.setImageResource(0); mProgressCircle.invalidate(); mController = new MeasurementController(getMainLooper(), new Device(this), new SensorFactory(this), mCameraPreview); mController.setListener(mListener); // server does not save then. maybe we do want to save... maybe create another test flag, for not saving? //if(AppInfo.getInstance(this).devSettings.skipDialogs) // mController.setUnderTest(true); mChart = new LineChartWrapper((LineChart) findViewById(R.id.chart)); final int PLOT_SAMPLES = (int) (FPS * 3.0); // see LineChartWrapper.WINDOW_SIZE_SECS // TODO(david): PlotFilter should be more robust, not just TOTAL_TIME mPlotFilter = new PlotFilter(FPS, (int) ((mTotalTime + 10.0) * FPS + 1), PLOT_SAMPLES); /* * when integrating RunningQuality into MeasurementController, keep in mind the necessity * of a separate background worker thread. */ mQuality = new RunningQuality(FPS); mQuality.setListener(new RunningQuality.BeatListener() { private int mBadBeatStartIndex = 0; @Override public void onLocked(int startIdx) { Log.i(TAG, "onLocked() at startIdx=" + startIdx); mNumNoisy = 0; mBadBeatStartIndex = 0; double qualityLockTime = mQuality.getTimeAt(startIdx); mController.setQualityLockTime(qualityLockTime); } @Override public void onBeat(int startIdx, boolean goodBeat, double posCorrCoeff) { //double time = mQuality.getTimeAt(startIdx); Log.i(TAG, "onBeat() SQI: posCorrCoeff=" + posCorrCoeff); if(!goodBeat) { mNumNoisy++; double t = mQuality.getTimeAt(startIdx); Log.i(TAG, "onBeat() NOISY BEAT"); Log.i(TAG, "onBeat() NOISY BEAT"); Log.i(TAG, "onBeat() NOISY BEAT <<<<<<<< noisy beat #" + mNumNoisy); Log.i(TAG, "onBeat() NOISY BEAT"); Log.i(TAG, "onBeat() NOISY BEAT"); } /* // ignored for doctors if(mNumNoisy > MAX_NUM_NOISY) { try { mController.stop(); } catch (IMeasurementController.FramesDroppedException e) { e.printStackTrace(); startErrorActivity(getString(R.string.frames_dropped)); return; } onMeasurementFailure(mController.getMeasurement()); } */ } }); mDataManager = MeasurementDataManager.getInstance(getApplicationContext()); mAppInfo = AppInfo.getInstance(this); } private class SignalAnimator extends Thread { private boolean running = true; private int i; SignalAnimator() { start(); } synchronized private boolean isRunning() { return running; } @Override public void run() { i = 0; while(isRunning()) { try { Thread.sleep(250); } catch (InterruptedException e) { e.printStackTrace(); break; } //mSignalIndicator.setImageResource(); final int ii = i; if(++i >= mTxSequence.length()) i = 0; runOnUiThread(new Runnable() { @Override public void run() { mSignalIndicator.setImageResource(mTxSequence.getResourceId(ii, -1)); mSignalIndicator.invalidate(); } }); } } synchronized void cancel() { running = false; interrupt(); try { join(); } catch (InterruptedException e) { e.printStackTrace(); } } } private SignalAnimator mAnimator; private void startAnimation() { if(mAnimator != null) return; // already animating mAnimator = new SignalAnimator(); } private void stopAnimation() { if(mAnimator != null) { mAnimator.cancel(); mAnimator = null; } } @Override protected void onResume() { super.onResume(); mUser = (UserMeta) getIntent().getSerializableExtra("meta"); mMode = (IMeasurementController.MeasurementMode) getIntent().getSerializableExtra("mode"); Log.i(TAG, "got age=" + mUser.age + " gender=" + mUser.gender + " mode=" + mMode); if(mMode == null) { // to do: e.g. 'Repeat measurement' button in ResultActivity Log.w(TAG, "should migrate all calling Activities to include mode"); mMode = IMeasurementController.MeasurementMode.RISK; } switch(mMode) { default: // fall through case RISK: mTotalTime = TOTAL_TIME_RISK; break; case ENDOTHELIAL_FUNCTION: mTotalTime = TOTAL_TIME_ENDOTHELIAL_FUNCTION; break; case VITAL_CHECK: mTotalTime = TOTAL_TIME_VITAL_CHECK; break; } mController.setMode(mMode); mController.setUnderTest(mDataManager.isUnderTest()); mPlotFilter.clear(); mQuality.clear(); mController.start(); if(mDataManager.getNumFailedMeasurements() > 0) mQuality.setBeatCorrThr2(RunningQuality.BEAT_CORR_THR_2_LOWER); else mQuality.setBeatCorrThr2(RunningQuality.BEAT_CORR_THR_2); mCountdownText.setText(Integer.toString((int) mTotalTime)); mProgressCircle.setImageResource(mProgSequence.getResourceId(0, -1)); mProgressCircle.invalidate(); mSignalIndicator.setImageResource(0); mProgressCircle.invalidate(); startBackgroundThread(); mDataManager.startBackgroundThread(); } private void onMeasurementFinished(Measurement measurement) { updateMeasurement(measurement); startAnimation(); // if dummy microphone, pretend we're already synced if(measurement.series.getAudio().length == 0) measurement.series.audioSynced = true; mDataManager.create(measurement, new IMeasurementDataManager.MeasurementResultListener() { @Override public void onResult(Measurement measurement) { //assert(measurement.result != null); // successful result from server Intent intent = new Intent(MeasureActivity.this, ResultActivity.class); intent.putExtra("measurementId", measurement.getId()); startActivity(intent); stopAnimation(); } @Override public void onError(Measurement measurement, IMeasurementDataManager.ErrorCode code, String message) { onCreateError(measurement, code, message); } }); } private void onCreateError(Measurement measurement, IMeasurementDataManager.ErrorCode code, String message) { // some error saving the stuff Toast.makeText(MeasureActivity.this, "Failed to get result: " + code + " " + message, Toast.LENGTH_LONG).show(); Log.e(TAG, "Failed to get result: " + code + " " + message); stopAnimation(); int failureReason; // note: UNCLASSIFIED is a regular, protocol-level response that is not an error. // hence it is not handled here, but displayed in ResultActivity in a special way switch(code) { case CONNECTION_ERROR: failureReason = R.drawable.signal_none; break; case HTTP_ERROR: case LOCAL_ERROR: case UNKNOWN_ERROR: case PROTOCOL_ERROR: failureReason = R.drawable.signal_fail; break; default: failureReason = R.drawable.signal_fail; } mSignalIndicator.setImageResource(failureReason); mSignalIndicator.invalidate(); } enum LogMessageType { MEASUREMENT_FAILURE, ERROR } private void onMeasurementFailure(Measurement measurement) { updateMeasurement(measurement); int numFailedBefore = mDataManager.getNumFailedMeasurements(); mDataManager.logMessage(measurement.appId, MEASUREMENT_FAILURE.toString(), "measurement failure. numFailedBefore=" + numFailedBefore); mDataManager.addFailedMeasurement(); startAnimation(); mDataManager.logFailure(measurement, new IMeasurementDataManager.MeasurementResultListener() { @Override public void onResult(Measurement measurement) { // show result from server (UNCLASSIFIED) Intent intent = new Intent(MeasureActivity.this, ResultActivity.class); intent.putExtra("measurementId", measurement.getId()); startActivity(intent); stopAnimation(); } @Override public void onError(Measurement measurement, IMeasurementDataManager.ErrorCode code, String message) { onCreateError(measurement, code, message); } }); } private void updateMeasurement(Measurement measurement) { measurement.user = mUser; measurement.app = mAppInfo.getAppMeta(); measurement.meta.mode = mMode; measurement.meta.test = TEST_MODE; // if dummy microphone, pretend we're already synced if(measurement.series.getAudio().length == 0) measurement.series.audioSynced = true; } @Override protected void onPause() { super.onPause(); try { mController.stop(); } catch (IMeasurementController.FramesDroppedException e) { // ignored, since result is discarded here anyways e.printStackTrace(); } stopAnimation(); stopBackgroundThread(); mDataManager.stopBackgroundThread(); } /** * Two layers of signal quality checks: * * 1. finger presence: (redness, uniformity, limit brightness-swings) * * restarts if you take off your finger * vibrates on restarts * * 30s timer limit if never locked for at least 10s * TODO: BUG: if never locked, timeSinceLock keeps increasing. * * 2. pulse beat quality: (beat detection and cross-correlation of beats) * * locks if 2+ similar beats are found * * only shows BPM if the quality is locked */ private RunningQuality mQuality; /** filter and plot the red channel of the camera */ private IMeasurementController.Listener mListener = new IMeasurementController.Listener() { private int mI; private IMeasurementController.State mPrevState = IMeasurementController.State.STOPPED; private double mTimeSinceLock = 0.0; private boolean mEverLocked = false; private void clear() { mI = 0; mPrevSigTs = 0.0; mTimeSinceLock = 0.0; efBuzzed1 = false; efBuzzed2 = false; } @Override public void onState(IMeasurementController.State newState) { if(newState == IMeasurementController.State.STOPPED || newState == IMeasurementController.State.FAILED) clear(); if(newState == IMeasurementController.State.FAILED) { if(mController.getFailureReason().equals(IMeasurementController.FailureReason.CAMERA_FRAMES_DROPPED)) { startErrorActivity(getString(R.string.frames_dropped)); } else { startErrorActivity("Error: Measurement failed: " + mController.getFailureReason().toString()); } } if(newState == IMeasurementController.State.SAMPLING) { if(!mEverLocked) { // short vibrate at first lock Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE); v.vibrate(200); } mEverLocked = true; } if(mPrevState == IMeasurementController.State.SAMPLING && newState == IMeasurementController.State.WAITING) { // avoid message when user puts their finger on (bouncing camera lock) if(mTimeSinceLock > 5.0) { Toast.makeText(MeasureActivity.this, "Measurement restarted", Toast.LENGTH_SHORT).show(); } mQuality.clear(); // note: both SQI and swing-level restarts go through here // vibrate on restart Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE); v.vibrate(200); } mPrevState = newState; } private double mPrevSigTs = 0.0; /** beatdetect() for SQI and BPM will run on the signal starting from (cameraLockTime + SQI_SETTLING_TIME) */ private static final double SQI_SETTLING_TIME = 3.0; private boolean efBuzzed1 = false; private boolean efBuzzed2 = false; @Override public void onFrame(final double timeSinceStart, final double timeSinceLock, double[] rgb, IntensityDetector.ImageSummary summary) { final int RED_CHANNEL = 0; final double entry = rgb[RED_CHANNEL]; Log.i(TAG, "ON_FRAME " + timeSinceStart + " " + timeSinceLock + " " + rgb[0] + " " + rgb[1] + " " + rgb[2]); mTimeSinceLock = timeSinceLock; if(++mI % 30 == 0) { mCountdownText.setText(Integer.toString(((int) (mTotalTime - timeSinceLock)))); int progSeqId = (int) (mProgSequence.length() * timeSinceLock / mTotalTime); progSeqId = Math.min(progSeqId, mProgSequence.length()-1); mProgressCircle.setImageResource(mProgSequence.getResourceId(progSeqId, -1)); mProgressCircle.invalidate(); // when pressing Back, it happens that mBackgroundHandler is null here - onPause() called before running this callback Runnable // we could check if Activity is paused before dispatching in MeasurementController, but then we'd need to hand the Activity to it. if(mBackgroundHandler != null) { if(mQuality.isRunning()) { mBackgroundHandler.post(new Runnable() { @Override public void run() { // takes some time, at longer signals (since it does full beat detection) // run it in the background to avoid UI lag of the graph plotting updateHeartRate(mController.getLockTime() + SQI_SETTLING_TIME); } }); } // // running signal quality // if(mPrevState == IMeasurementController.State.SAMPLING && timeSinceLock > SQI_SETTLING_TIME) { mBackgroundHandler.post(new Runnable() { @Override public void run() { // use signal since lock, otherwise the amplitude in Signal.beatdetect() takes too long to settle if(mPrevSigTs == 0.0) mPrevSigTs = mController.getLockTime() + SQI_SETTLING_TIME; // must use detrended data, since the DC would mean the correlations will look too good to be true. PlotFilter.Series ser = getSeriesSinceLock(mPrevSigTs); if(ser.t.length > 0) { mQuality.append(ser.t[0], ser.x); Log.i(TAG, "mQuality.append() of pts.length=" + ser.x.length); mPrevSigTs = ser.t[ser.t.length - 1]; } } }); } } } // note: if the camera was never locked (unlikely), `timeSinceLock` reports same as `timeSinceStart` //boolean lockOverdue = timeSinceLock > REQUIRE_LOCK_BY_TIME; // camera lock <--- long time ---> ? boolean lockOverdue = false; // disabled for doctors - always finish the measurement. if(lockOverdue && !mQuality.isRunning()) { try { mController.stop(); } catch (IMeasurementController.FramesDroppedException e) { e.printStackTrace(); startErrorActivity(getString(R.string.frames_dropped)); return; } onMeasurementFailure(mController.getMeasurement()); return; } // bad: since more than 30 secs, there was no lock of over 10 sec duration? boolean finished = timeSinceLock > mTotalTime; boolean notLockedTrouble = (timeSinceStart > 30.0 && timeSinceLock < 10.0); // && (mMode.equals(IMeasurementController.MeasurementMode.RISK) || (mMode.equals(IMeasurementController.MeasurementMode.ENDOTHELIAL_FUNCTION) && timeSinceStart < TOTAL_TIME_RISK)) if(mMode.equals(IMeasurementController.MeasurementMode.ENDOTHELIAL_FUNCTION) && timeSinceLock > TOTAL_TIME_RISK && !efBuzzed1) { // buzz 1: inflate cuff (provoke arm ischemia) Toast.makeText(MeasureActivity.this, "Inflate cuff now", Toast.LENGTH_LONG).show(); Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE); v.vibrate(200); efBuzzed1 = true; } else if(mMode.equals(IMeasurementController.MeasurementMode.ENDOTHELIAL_FUNCTION) && timeSinceLock > (TOTAL_TIME_ENDOTHELIAL_FUNCTION - TOTAL_TIME_RISK) && !efBuzzed2) { // buzz 2: deflate cuff (remove arm ischemia) Toast.makeText(MeasureActivity.this, "Deflate cuff now", Toast.LENGTH_LONG).show(); Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE); v.vibrate(200); efBuzzed2 = true; } if(finished || notLockedTrouble) { // ensure we only call stop() once if(mController.getState() == IMeasurementController.State.STOPPED) return; try { Log.i(TAG, "since more than 30 secs, there was no lock of over 10 sec duration. timeSinceLock=" + timeSinceLock + " timeSinceStart=" + timeSinceStart); mController.stop(); if(mQuality.isRunning() && mNumNoisy <= MAX_NUM_NOISY) { // what if never locked properly? wrong API endpoint? onMeasurementFinished(mController.getMeasurement()); } else { // long vibrate on fail Vibrator v = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE); v.vibrate(new long[]{400, 400, 400, 400}, -1); // codepath used for doctors onMeasurementFailure(mController.getMeasurement()); } } catch (IMeasurementController.FramesDroppedException e) { e.printStackTrace(); startErrorActivity(getString(R.string.frames_dropped)); } return; } final Runnable runnable = new Runnable() { @Override public void run() { mPlotFilter.addUneven(timeSinceStart, entry); PlotFilter.Series slice = mPlotFilter.getPlotSeries(); mChart.replaceData(slice.t, slice.x); } }; runOnUiThread(runnable); } }; private void startOtherActivity(Class activityClass) { Intent intent = new Intent(MeasureActivity.this, activityClass); intent.putExtra("meta", mUser); startActivity(intent); stopAnimation(); } private void startErrorActivity(String message) { mDataManager.logMessage(mAppInfo.getAppMeta().appId, ERROR.toString(), message); Intent intent = new Intent(MeasureActivity.this, ErrorActivity.class); intent.putExtra("meta", mUser); intent.putExtra("message", message); startActivity(intent); stopAnimation(); } private PlotFilter.Series getSeriesSinceLock(double lockTime) { int len = mPlotFilter.size(); PlotFilter.Series series = mPlotFilter.getRecentSeries(len); int istart = 0; for(; istart < series.t.length; istart++) if(series.t[istart] > lockTime) break; double[] x = DVec.get(series.x, istart, series.x.length); double[] t = DVec.get(series.t, istart, series.t.length); return new PlotFilter.Series(t, x); } private void updateHeartRate(double sinceTime) { int len = mPlotFilter.size(); // ignore the initial, large swing PlotFilter.Series ser = getSeriesSinceLock(sinceTime); double[] pts = ser.x; // we should make sure that detrended amplitude swing is much smaller (< e.g. 16, see ipynb exploration) in recent stuff Log.i(TAG, "updateHeartRate() len=" + len + " pointsSinceLock.length=" + pts.length); final Double bpm = Signal.calculateBpm(FPS, pts); if(pts.length >= 60) { double[] smoothd = Signal.running_mean(Signal.running_mean(pts, 6), 7); double[] slice = DVec.get(smoothd, smoothd.length - 30, smoothd.length); Log.i(TAG, "swing amplitude: " + (DVec.max(slice) - DVec.min(slice))); } runOnUiThread(new Runnable() { @Override public void run() { if(bpm != null) mHeartRateText.setText(Integer.toString(bpm.intValue())); else mHeartRateText.setText("- "); } }); } private void startBackgroundThread() { mBackgroundThread = new HandlerThread("MeasureBackground"); mBackgroundThread.start(); mBackgroundHandler = new Handler(mBackgroundThread.getLooper()); } private void stopBackgroundThread() { if(mBackgroundThread != null) { mBackgroundThread.quitSafely(); try { mBackgroundThread.join(); mBackgroundThread = null; mBackgroundHandler = null; } catch (InterruptedException e) { e.printStackTrace(); } } } }