fix: android, controlled side, gesture (#10792)

Signed-off-by: fufesou <linlong1266@gmail.com>

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -34,9 +34,9 @@ import hbb.MessageOuterClass.KeyEvent
 import hbb.MessageOuterClass.KeyboardMode
 import hbb.KeyEventConverter
 
-const val LIFT_DOWN = 9
+const val LEFT_DOWN = 9
-const val LIFT_MOVE = 8
+const val LEFT_MOVE = 8
-const val LIFT_UP = 10
+const val LEFT_UP = 10
 const val RIGHT_UP = 18
 const val WHEEL_BUTTON_DOWN = 33
 const val WHEEL_BUTTON_UP = 34
@@ -65,6 +65,7 @@ class InputService : AccessibilityService() {
     private val logTag = "input service"
     private var leftIsDown = false
     private var touchPath = Path()
+    private var stroke: GestureDescription.StrokeDescription? = null
     private var lastTouchGestureStartTime = 0L
     private var mouseX = 0
     private var mouseY = 0
@@ -77,6 +78,9 @@ class InputService : AccessibilityService() {
 
     private var fakeEditTextForTextStateCalculation: EditText? = null
 
+    private var lastX = 0
+    private var lastY = 0
+
     private val volumeController: VolumeController by lazy { VolumeController(applicationContext.getSystemService(AUDIO_SERVICE) as AudioManager) }
 
     @RequiresApi(Build.VERSION_CODES.N)
@@ -84,7 +88,7 @@ class InputService : AccessibilityService() {
         val x = max(0, _x)
         val y = max(0, _y)
 
-        if (mask == 0 || mask == LIFT_MOVE) {
+        if (mask == 0 || mask == LEFT_MOVE) {
             val oldX = mouseX
             val oldY = mouseY
             mouseX = x * SCREEN_INFO.scale
@@ -99,14 +103,13 @@ class InputService : AccessibilityService() {
         }
 
         // left button down ,was up
-        if (mask == LIFT_DOWN) {
+        if (mask == LEFT_DOWN) {
             isWaitingLongPress = true
             timer.schedule(object : TimerTask() {
                 override fun run() {
                     if (isWaitingLongPress) {
                         isWaitingLongPress = false
-                        leftIsDown = false
+                        continueGesture(mouseX, mouseY)
-                        endGesture(mouseX, mouseY)
                     }
                 }
             }, LONG_TAP_DELAY * 4)
@@ -122,7 +125,7 @@ class InputService : AccessibilityService() {
         }
 
         // left up ,was down
-        if (mask == LIFT_UP) {
+        if (mask == LEFT_UP) {
             if (leftIsDown) {
                 leftIsDown = false
                 isWaitingLongPress = false
@@ -242,35 +245,57 @@ class InputService : AccessibilityService() {
     }
 
     private fun startGesture(x: Int, y: Int) {
-        touchPath = Path()
+        touchPath.reset()
         touchPath.moveTo(x.toFloat(), y.toFloat())
         lastTouchGestureStartTime = System.currentTimeMillis()
-    }
+        lastX = x
-
+        lastY = y
-    private fun continueGesture(x: Int, y: Int) {
-        touchPath.lineTo(x.toFloat(), y.toFloat())
     }
 
     @RequiresApi(Build.VERSION_CODES.N)
-    private fun endGesture(x: Int, y: Int) {
+    private fun doDispatchGesture(x: Int, y: Int, willContinue: Boolean) {
-        try {
         touchPath.lineTo(x.toFloat(), y.toFloat())
         var duration = System.currentTimeMillis() - lastTouchGestureStartTime
         if (duration <= 0) {
             duration = 1
         }
-            val stroke = GestureDescription.StrokeDescription(
+        try {
+            if (stroke == null) {
+                stroke = GestureDescription.StrokeDescription(
                     touchPath,
                     0,
-                duration
+                    duration,
+                    willContinue
                 )
+            } else {
+                stroke = stroke?.continueStroke(touchPath, 0, duration, willContinue)
+            }
+            stroke?.let {
                 val builder = GestureDescription.Builder()
-            builder.addStroke(stroke)
+                builder.addStroke(it)
                 Log.d(logTag, "end gesture x:$x y:$y time:$duration")
                 dispatchGesture(builder.build(), null, null)
-        } catch (e: Exception) {
-            Log.e(logTag, "endGesture error:$e")
             }
+        } catch (e: Exception) {
+            Log.e(logTag, "doDispatchGesture, willContinue:$willContinue, error:$e")
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.N)
+    private fun continueGesture(x: Int, y: Int) {
+        doDispatchGesture(x, y, true)
+        touchPath.reset()
+        touchPath.moveTo(x.toFloat(), y.toFloat())
+        lastTouchGestureStartTime = System.currentTimeMillis()
+        lastX = x
+        lastY = y
+    }
+
+    @RequiresApi(Build.VERSION_CODES.N)
+    private fun endGesture(x: Int, y: Int) {
+        doDispatchGesture(x, y, false)
+        touchPath.reset()
+        stroke = null
     }
 
     @RequiresApi(Build.VERSION_CODES.N)

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -65,8 +65,8 @@ class MainService : Service() {
     @Keep
     @RequiresApi(Build.VERSION_CODES.N)
     fun rustPointerInput(kind: Int, mask: Int, x: Int, y: Int) {
-        // turn on screen with LIFT_DOWN when screen off
+        // turn on screen with LEFT_DOWN when screen off
-        if (!powerManager.isInteractive && (kind == 0 || mask == LIFT_DOWN)) {
+        if (!powerManager.isInteractive && (kind == 0 || mask == LEFT_DOWN)) {
             if (wakeLock.isHeld) {
                 Log.d(logTag, "Turn on Screen, WakeLock release")
                 wakeLock.release()

flutter/lib/common/widgets/remote_input.dart

@@ -187,6 +187,11 @@ class _RawTouchGestureDetectorRegionState
         return;
       }
       _cacheLongPressPositionTs = DateTime.now().millisecondsSinceEpoch;
+      if (ffiModel.isPeerMobile) {
+        await ffi.cursorModel
+            .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy);
+        await inputModel.tapDown(MouseButtons.left);
+      }
     }
   }
 
@@ -204,6 +209,7 @@ class _RawTouchGestureDetectorRegionState
     if (lastDeviceKind != PointerDeviceKind.touch) {
       return;
     }
+    if (!ffi.ffiModel.isPeerMobile) {
       if (handleTouch) {
         final isMoved = await ffi.cursorModel
             .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy);
@@ -211,8 +217,23 @@ class _RawTouchGestureDetectorRegionState
           return;
         }
       }
-    if (!ffi.ffiModel.isPeerMobile) {
       await inputModel.tap(MouseButtons.right);
+    } else {
+      // It's better to send a message to tell the controlled device that the long press event is triggered.
+      // We're now using a `TimerTask` in `InputService.kt` to decide whether to trigger the long press event.
+      // It's not accurate and it's better to use the same detection logic in the controlling side.
+    }
+  }
+
+  onLongPressMoveUpdate(LongPressMoveUpdateDetails d) async {
+    if (!ffiModel.isPeerMobile || lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      if (!ffi.cursorModel.isInRemoteRect(d.localPosition)) {
+        return;
+      }
+      await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
     }
   }
 
@@ -432,7 +453,8 @@ class _RawTouchGestureDetectorRegionState
         instance
           ..onLongPressDown = onLongPressDown
           ..onLongPressUp = onLongPressUp
-          ..onLongPress = onLongPress;
+          ..onLongPress = onLongPress
+          ..onLongPressMoveUpdate = onLongPressMoveUpdate;
       }),
       // Customized
       HoldTapMoveGestureRecognizer: