Merge pull request #5323 from dignow/refact/android_scroll_event

Refact/android scroll event

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -26,6 +26,13 @@ const val WHEEL_BUTTON_UP = 34
 const val WHEEL_DOWN = 523331
 const val WHEEL_UP = 963
 
+const val TOUCH_SCALE_START = 1
+const val TOUCH_SCALE = 2
+const val TOUCH_SCALE_END = 3
+const val TOUCH_PAN_START = 4
+const val TOUCH_PAN_UPDATE = 5
+const val TOUCH_PAN_END = 6
+
 const val WHEEL_STEP = 120
 const val WHEEL_DURATION = 50L
 const val LONG_TAP_DELAY = 200L
@@ -167,6 +174,30 @@ class InputService : AccessibilityService() {
         }
     }
 
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun onTouchInput(mask: Int, _x: Int, _y: Int) {
+        when (mask) {
+            TOUCH_PAN_UPDATE -> {
+                mouseX -= _x * SCREEN_INFO.scale
+                mouseY -= _y * SCREEN_INFO.scale
+                mouseX = max(0, mouseX);
+                mouseY = max(0, mouseY);
+                continueGesture(mouseX, mouseY)
+            }
+            TOUCH_PAN_START -> {
+                mouseX = max(0, _x) * SCREEN_INFO.scale
+                mouseY = max(0, _y) * SCREEN_INFO.scale
+                startGesture(mouseX, mouseY)
+            }
+            TOUCH_PAN_END -> {
+                endGesture(mouseX, mouseY)
+                mouseX = max(0, _x) * SCREEN_INFO.scale
+                mouseY = max(0, _y) * SCREEN_INFO.scale
+            }
+            else -> {}
+        }
+    }
+
     @RequiresApi(Build.VERSION_CODES.N)
     private fun consumeWheelActions() {
         if (isWheelActionsPolling) {

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -71,17 +71,26 @@ class MainService : Service() {
 
     @Keep
     @RequiresApi(Build.VERSION_CODES.N)
-    fun rustMouseInput(mask: Int, x: Int, y: Int) {
+    fun rustPointerInput(kind: String, mask: Int, x: Int, y: Int) {
         // turn on screen with LIFT_DOWN when screen off
-        if (!powerManager.isInteractive && mask == LIFT_DOWN) {
+        if (!powerManager.isInteractive && (kind == "touch" || mask == LIFT_DOWN)) {
             if (wakeLock.isHeld) {
-                Log.d(logTag,"Turn on Screen, WakeLock release")
+                Log.d(logTag, "Turn on Screen, WakeLock release")
                 wakeLock.release()
             }
             Log.d(logTag,"Turn on Screen")
             wakeLock.acquire(5000)
         } else {
-            InputService.ctx?.onMouseInput(mask,x,y)
+            when (kind) {
+                "touch" -> {
+                    InputService.ctx?.onTouchInput(mask, x, y)
+                }
+                "mouse" -> {
+                    InputService.ctx?.onMouseInput(mask, x, y)
+                }
+                else -> {
+                }
+            }
         }
     }
 

flutter/lib/common/widgets/remote_input.dart

@@ -6,6 +6,7 @@ import 'package:flutter/gestures.dart';
 
 import 'package:flutter_hbb/models/platform_model.dart';
 import 'package:flutter_hbb/common.dart';
+import 'package:flutter_hbb/consts.dart';
 import 'package:flutter_hbb/models/model.dart';
 import 'package:flutter_hbb/models/input_model.dart';
 
@@ -263,9 +264,9 @@ class _RawTouchGestureDetectorRegionState
       if (scale != 0) {
         bind.sessionSendPointer(
             sessionId: sessionId,
-            msg: json.encode({
-              'touch': {'scale': scale}
-            }));
+            msg: json.encode(
+                PointerEventToRust(kPointerEventKindTouch, 'scale', scale)
+                    .toJson()));
       }
     } else {
       // mobile
@@ -283,9 +284,8 @@ class _RawTouchGestureDetectorRegionState
     if (isDesktop) {
       bind.sessionSendPointer(
           sessionId: sessionId,
-          msg: json.encode({
-            'touch': {'scale': 0}
-          }));
+          msg: json.encode(
+              PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson()));
     } else {
       // mobile
       _scale = 1;

flutter/lib/consts.dart

@@ -54,6 +54,9 @@ const String kTabLabelSettingPage = "Settings";
 const String kWindowPrefix = "wm_";
 const int kWindowMainId = 0;
 
+const String kPointerEventKindTouch = "touch";
+const String kPointerEventKindMouse = "mouse";
+
 // the executable name of the portable version
 const String kEnvPortableExecutable = "RUSTDESK_APPNAME";
 

flutter/lib/models/input_model.dart

@@ -35,6 +35,24 @@ extension ToString on MouseButtons {
   }
 }
 
+class PointerEventToRust {
+  final String kind;
+  final String type;
+  final dynamic value;
+
+  PointerEventToRust(this.kind, this.type, this.value);
+
+  Map<String, dynamic> toJson() {
+    return {
+      'k': kind,
+      'v': {
+        't': type,
+        'v': value,
+      }
+    };
+  }
+}
+
 class InputModel {
   final WeakReference<FFI> parent;
   String keyboardMode = "legacy";
@@ -62,11 +80,11 @@ class InputModel {
   int _lastButtons = 0;
   Offset lastMousePos = Offset.zero;
 
-  get id => parent.target?.id ?? "";
-
   late final SessionID sessionId;
 
   bool get keyboardPerm => parent.target!.ffiModel.keyboard;
+  String get id => parent.target?.id ?? '';
+  String? get peerPlatform => parent.target?.ffiModel.pi.platform;
 
   InputModel(this.parent) {
     sessionId = parent.target!.sessionId;
@@ -223,14 +241,8 @@ class InputModel {
         command: command);
   }
 
-  Map<String, dynamic> getEvent(PointerEvent evt, String type) {
+  Map<String, dynamic> _getMouseEvent(PointerEvent evt, String type) {
     final Map<String, dynamic> out = {};
-    out['x'] = evt.position.dx;
-    out['y'] = evt.position.dy;
-    if (alt) out['alt'] = 'true';
-    if (shift) out['shift'] = 'true';
-    if (ctrl) out['ctrl'] = 'true';
-    if (command) out['command'] = 'true';
 
     // Check update event type and set buttons to be sent.
     int buttons = _lastButtons;
@@ -260,7 +272,6 @@ class InputModel {
 
     out['buttons'] = buttons;
     out['type'] = type;
-
     return out;
   }
 
@@ -292,7 +303,7 @@ class InputModel {
   }
 
   /// Modify the given modifier map [evt] based on current modifier key status.
-  Map<String, String> modify(Map<String, String> evt) {
+  Map<String, dynamic> modify(Map<String, dynamic> evt) {
     if (ctrl) evt['ctrl'] = 'true';
     if (shift) evt['shift'] = 'true';
     if (alt) evt['alt'] = 'true';
@@ -334,27 +345,33 @@ class InputModel {
       isPhysicalMouse.value = true;
     }
     if (isPhysicalMouse.value) {
-      handleMouse(getEvent(e, _kMouseEventMove));
+      handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position);
     }
   }
 
   void onPointerPanZoomStart(PointerPanZoomStartEvent e) {
     _lastScale = 1.0;
     _stopFling = true;
+
+    if (peerPlatform == kPeerPlatformAndroid) {
+      handlePointerEvent('touch', 'pan_start', e.position);
+    }
   }
 
   // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures
   void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) {
-    final scale = ((e.scale - _lastScale) * 1000).toInt();
-    _lastScale = e.scale;
+    if (peerPlatform != kPeerPlatformAndroid) {
+      final scale = ((e.scale - _lastScale) * 1000).toInt();
+      _lastScale = e.scale;
 
-    if (scale != 0) {
-      bind.sessionSendPointer(
-          sessionId: sessionId,
-          msg: json.encode({
-            'touch': {'scale': scale}
-          }));
-      return;
+      if (scale != 0) {
+        bind.sessionSendPointer(
+            sessionId: sessionId,
+            msg: json.encode(
+                PointerEventToRust(kPointerEventKindTouch, 'scale', scale)
+                    .toJson()));
+        return;
+      }
     }
 
     final delta = e.panDelta;
@@ -362,7 +379,7 @@ class InputModel {
 
     var x = delta.dx.toInt();
     var y = delta.dy.toInt();
-    if (parent.target?.ffiModel.pi.platform == kPeerPlatformLinux) {
+    if (peerPlatform == kPeerPlatformLinux) {
       _trackpadScrollUnsent += (delta * _trackpadSpeed);
       x = _trackpadScrollUnsent.dx.truncate();
       y = _trackpadScrollUnsent.dy.truncate();
@@ -378,9 +395,13 @@ class InputModel {
       }
     }
     if (x != 0 || y != 0) {
-      bind.sessionSendMouse(
-          sessionId: sessionId,
-          msg: '{"type": "trackpad", "x": "$x", "y": "$y"}');
+      if (peerPlatform == kPeerPlatformAndroid) {
+        handlePointerEvent('touch', 'pan_update', Offset(x.toDouble(), y.toDouble()));
+      } else {
+        bind.sessionSendMouse(
+            sessionId: sessionId,
+            msg: '{"type": "trackpad", "x": "$x", "y": "$y"}');
+      }
     }
   }
 
@@ -436,11 +457,15 @@ class InputModel {
   }
 
   void onPointerPanZoomEnd(PointerPanZoomEndEvent e) {
+    if (peerPlatform == kPeerPlatformAndroid) {
+      handlePointerEvent('touch', 'pan_end', e.position);
+      return;
+    }
+
     bind.sessionSendPointer(
         sessionId: sessionId,
-        msg: json.encode({
-          'touch': {'scale': 0}
-        }));
+        msg: json.encode(
+            PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson()));
 
     waitLastFlingDone();
     _stopFling = false;
@@ -465,21 +490,21 @@ class InputModel {
       }
     }
     if (isPhysicalMouse.value) {
-      handleMouse(getEvent(e, _kMouseEventDown));
+      handleMouse(_getMouseEvent(e, _kMouseEventDown), e.position);
     }
   }
 
   void onPointUpImage(PointerUpEvent e) {
     if (e.kind != ui.PointerDeviceKind.mouse) return;
     if (isPhysicalMouse.value) {
-      handleMouse(getEvent(e, _kMouseEventUp));
+      handleMouse(_getMouseEvent(e, _kMouseEventUp), e.position);
     }
   }
 
   void onPointMoveImage(PointerMoveEvent e) {
     if (e.kind != ui.PointerDeviceKind.mouse) return;
     if (isPhysicalMouse.value) {
-      handleMouse(getEvent(e, _kMouseEventMove));
+      handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position);
     }
   }
 
@@ -504,19 +529,16 @@ class InputModel {
   }
 
   void refreshMousePos() => handleMouse({
-        'x': lastMousePos.dx,
-        'y': lastMousePos.dy,
         'buttons': 0,
         'type': _kMouseEventMove,
-      });
+      }, lastMousePos);
 
   void tryMoveEdgeOnExit(Offset pos) => handleMouse(
         {
-          'x': pos.dx,
-          'y': pos.dy,
           'buttons': 0,
           'type': _kMouseEventMove,
         },
+        pos,
         onExit: true,
       );
 
@@ -550,17 +572,49 @@ class InputModel {
     return Offset(x, y);
   }
 
-  void handleMouse(
-    Map<String, dynamic> evt, {
-    bool onExit = false,
-  }) {
-    double x = evt['x'];
-    double y = max(0.0, evt['y']);
-    final cursorModel = parent.target!.cursorModel;
+  void handlePointerEvent(String kind, String type, Offset offset) {
+    double x = offset.dx;
+    double y = offset.dy;
+    if (_checkPeerControlProtected(x, y)) {
+      return;
+    }
+    // Only touch events are handled for now. So we can just ignore buttons.
+    // to-do: handle mouse events
 
+    late final dynamic evtValue;
+    if (type == 'pan_update') {
+      evtValue = {
+        'x': x.toInt(),
+        'y': y.toInt(),
+      };
+    } else {
+      final isMoveTypes = ['pan_start', 'pan_end'];
+      final pos = handlePointerDevicePos(
+        kPointerEventKindTouch,
+        x,
+        y,
+        isMoveTypes.contains(type),
+        type,
+      );
+      if (pos == null) {
+        return;
+      }
+      evtValue = {
+        'x': pos.x,
+        'y': pos.y,
+      };
+    }
+
+    final evt = PointerEventToRust(kind, type, evtValue).toJson();
+    bind.sessionSendPointer(
+        sessionId: sessionId, msg: json.encode(modify(evt)));
+  }
+
+  bool _checkPeerControlProtected(double x, double y) {
+    final cursorModel = parent.target!.cursorModel;
     if (cursorModel.isPeerControlProtected) {
       lastMousePos = ui.Offset(x, y);
-      return;
+      return true;
     }
 
     if (!cursorModel.gotMouseControl) {
@@ -571,10 +625,23 @@ class InputModel {
         cursorModel.gotMouseControl = true;
       } else {
         lastMousePos = ui.Offset(x, y);
-        return;
+        return true;
       }
     }
     lastMousePos = ui.Offset(x, y);
+    return false;
+  }
+
+  void handleMouse(
+    Map<String, dynamic> evt,
+    Offset offset, {
+    bool onExit = false,
+  }) {
+    double x = offset.dx;
+    double y = max(0.0, offset.dy);
+    if (_checkPeerControlProtected(x, y)) {
+      return;
+    }
 
     var type = '';
     var isMove = false;
@@ -592,17 +659,58 @@ class InputModel {
         return;
     }
     evt['type'] = type;
+
+    final pos = handlePointerDevicePos(
+      kPointerEventKindMouse,
+      x,
+      y,
+      isMove,
+      type,
+      onExit: onExit,
+      buttons: evt['buttons'],
+    );
+    if (pos == null) {
+      return;
+    }
+    if (type != '') {
+      evt['x'] = '0';
+      evt['y'] = '0';
+    } else {
+      evt['x'] = '${pos.x}';
+      evt['y'] = '${pos.y}';
+    }
+
+    Map<int, String> mapButtons = {
+      kPrimaryMouseButton: 'left',
+      kSecondaryMouseButton: 'right',
+      kMiddleMouseButton: 'wheel',
+      kBackMouseButton: 'back',
+      kForwardMouseButton: 'forward'
+    };
+    evt['buttons'] = mapButtons[evt['buttons']] ?? '';
+    bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt)));
+  }
+
+  Point? handlePointerDevicePos(
+    String kind,
+    double x,
+    double y,
+    bool isMove,
+    String evtType, {
+    bool onExit = false,
+    int buttons = kPrimaryMouseButton,
+  }) {
     y -= CanvasModel.topToEdge;
     x -= CanvasModel.leftToEdge;
     final canvasModel = parent.target!.canvasModel;
-    final nearThr = 3;
-    var nearRight = (canvasModel.size.width - x) < nearThr;
-    var nearBottom = (canvasModel.size.height - y) < nearThr;
-
     final ffiModel = parent.target!.ffiModel;
     if (isMove) {
       canvasModel.moveDesktopMouse(x, y);
     }
+
+    final nearThr = 3;
+    var nearRight = (canvasModel.size.width - x) < nearThr;
+    var nearBottom = (canvasModel.size.height - y) < nearThr;
     final d = ffiModel.display;
     final imageWidth = d.width * canvasModel.scale;
     final imageHeight = d.height * canvasModel.scale;
@@ -650,7 +758,7 @@ class InputModel {
     } catch (e) {
       debugPrintStack(
           label: 'canvasModel.scale value ${canvasModel.scale}, $e');
-      return;
+      return null;
     }
 
     int minX = d.x.toInt();
@@ -659,40 +767,16 @@ class InputModel {
     int maxY = (d.y + d.height).toInt() - 1;
     evtX = trySetNearestRange(evtX, minX, maxX, 5);
     evtY = trySetNearestRange(evtY, minY, maxY, 5);
-    if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) {
-      // If left mouse up, no early return.
-      if (evt['buttons'] != kPrimaryMouseButton || type != 'up') {
-        return;
+    if (kind == kPointerEventKindMouse) {
+      if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) {
+        // If left mouse up, no early return.
+        if (!(buttons == kPrimaryMouseButton && evtType == 'up')) {
+          return null;
+        }
       }
     }
 
-    if (type != '') {
-      evtX = 0;
-      evtY = 0;
-    }
-
-    evt['x'] = '$evtX';
-    evt['y'] = '$evtY';
-    var buttons = '';
-    switch (evt['buttons']) {
-      case kPrimaryMouseButton:
-        buttons = 'left';
-        break;
-      case kSecondaryMouseButton:
-        buttons = 'right';
-        break;
-      case kMiddleMouseButton:
-        buttons = 'wheel';
-        break;
-      case kBackMouseButton:
-        buttons = 'back';
-        break;
-      case kForwardMouseButton:
-        buttons = 'forward';
-        break;
-    }
-    evt['buttons'] = buttons;
-    bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt));
+    return Point(evtX, evtY);
   }
 
   /// Web only

libs/hbb_common/protos/message.proto

@@ -118,9 +118,29 @@ message TouchScaleUpdate {
   int32 scale = 1;
 }
 
+message TouchPanStart {
+  int32 x = 1;
+  int32 y = 2;
+}
+
+message TouchPanUpdate {
+  // The delta x position relative to the previous position.
+  int32 x = 1;
+  // The delta y position relative to the previous position.
+  int32 y = 2;
+}
+
+message TouchPanEnd {
+  int32 x = 1;
+  int32 y = 2;
+}
+
 message TouchEvent {
   oneof union {
     TouchScaleUpdate scale_update = 1;
+    TouchPanStart pan_start = 2;
+    TouchPanUpdate pan_update = 3;
+    TouchPanEnd pan_end = 4;
   }
 }
 

libs/scrap/src/android/ffi.rs

@@ -154,17 +154,18 @@ pub extern "system" fn Java_com_carriez_flutter_1hbb_MainService_init(
     }
 }
 
-pub fn call_main_service_mouse_input(mask: i32, x: i32, y: i32) -> JniResult<()> {
+pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> {
     if let (Some(jvm), Some(ctx)) = (
         JVM.read().unwrap().as_ref(),
         MAIN_SERVICE_CTX.read().unwrap().as_ref(),
     ) {
         let mut env = jvm.attach_current_thread_as_daemon()?;
+        let kind = env.new_string(kind)?;
         env.call_method(
             ctx,
-            "rustMouseInput",
-            "(III)V",
-            &[JValue::Int(mask), JValue::Int(x), JValue::Int(y)],
+            "rustPointerInput",
+            "(Ljava/lang/String;III)V",
+            &[JValue::Object(&JObject::from(kind)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)],
         )?;
         return Ok(());
     } else {

src/flutter.rs

@@ -1130,6 +1130,85 @@ pub fn stop_global_event_stream(app_type: String) {
     let _ = GLOBAL_EVENT_STREAM.write().unwrap().remove(&app_type);
 }
 
+#[inline]
+fn session_send_touch_scale(
+    session_id: SessionID,
+    v: &serde_json::Value,
+    alt: bool,
+    ctrl: bool,
+    shift: bool,
+    command: bool,
+) {
+    match v.get("v").and_then(|s| s.as_i64()) {
+        Some(scale) => {
+            if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
+                session.send_touch_scale(scale as _, alt, ctrl, shift, command);
+            }
+        }
+        None => {}
+    }
+}
+
+#[inline]
+fn session_send_touch_pan(
+    session_id: SessionID,
+    v: &serde_json::Value,
+    pan_event: &str,
+    alt: bool,
+    ctrl: bool,
+    shift: bool,
+    command: bool,
+) {
+    match v.get("v") {
+        Some(v) => match (
+            v.get("x").and_then(|x| x.as_i64()),
+            v.get("y").and_then(|y| y.as_i64()),
+        ) {
+            (Some(x), Some(y)) => {
+                if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
+                    session
+                        .send_touch_pan_event(pan_event, x as _, y as _, alt, ctrl, shift, command);
+                }
+            }
+            _ => {}
+        },
+        _ => {}
+    }
+}
+
+fn session_send_touch_event(
+    session_id: SessionID,
+    v: &serde_json::Value,
+    alt: bool,
+    ctrl: bool,
+    shift: bool,
+    command: bool,
+) {
+    match v.get("t").and_then(|t| t.as_str()) {
+        Some("scale") => session_send_touch_scale(session_id, v, alt, ctrl, shift, command),
+        Some(pan_event) => {
+            session_send_touch_pan(session_id, v, pan_event, alt, ctrl, shift, command)
+        }
+        _ => {}
+    }
+}
+
+pub fn session_send_pointer(session_id: SessionID, msg: String) {
+    if let Ok(m) = serde_json::from_str::<HashMap<String, serde_json::Value>>(&msg) {
+        let alt = m.get("alt").is_some();
+        let ctrl = m.get("ctrl").is_some();
+        let shift = m.get("shift").is_some();
+        let command = m.get("command").is_some();
+        match (m.get("k"), m.get("v")) {
+            (Some(k), Some(v)) => match k.as_str() {
+                Some("touch") => session_send_touch_event(session_id, v, alt, ctrl, shift, command),
+                _ => {}
+            },
+            _ => {}
+        }
+    }
+}
+
 #[no_mangle]
 unsafe extern "C" fn get_rgba() {}
 

src/flutter_ffi.rs

@@ -1179,21 +1179,7 @@ pub fn main_load_ab() -> String {
 }
 
 pub fn session_send_pointer(session_id: SessionID, msg: String) {
-    if let Ok(m) = serde_json::from_str::<HashMap<String, serde_json::Value>>(&msg) {
-        let alt = m.get("alt").is_some();
-        let ctrl = m.get("ctrl").is_some();
-        let shift = m.get("shift").is_some();
-        let command = m.get("command").is_some();
-        if let Some(touch_event) = m.get("touch") {
-            if let Some(scale) = touch_event.get("scale") {
-                if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
-                    if let Some(scale) = scale.as_i64() {
-                        session.send_touch_scale(scale as _, alt, ctrl, shift, command);
-                    }
-                }
-            }
-        }
-    }
+    super::flutter::session_send_pointer(session_id, msg);
 }
 
 pub fn session_send_mouse(session_id: SessionID, msg: String) {

src/server/connection.rs

@@ -39,7 +39,7 @@ use hbb_common::{
     tokio_util::codec::{BytesCodec, Framed},
 };
 #[cfg(any(target_os = "android", target_os = "ios"))]
-use scrap::android::call_main_service_mouse_input;
+use scrap::android::call_main_service_pointer_input;
 use serde_json::{json, value::Value};
 use sha2::{Digest, Sha256};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -1547,8 +1547,8 @@ impl Connection {
             match msg.union {
                 Some(message::Union::MouseEvent(me)) => {
                     #[cfg(any(target_os = "android", target_os = "ios"))]
-                    if let Err(e) = call_main_service_mouse_input(me.mask, me.x, me.y) {
-                        log::debug!("call_main_service_mouse_input fail:{}", e);
+                    if let Err(e) = call_main_service_pointer_input("mouse", me.mask, me.x, me.y) {
+                        log::debug!("call_main_service_pointer_input fail:{}", e);
                     }
                     #[cfg(not(any(target_os = "android", target_os = "ios")))]
                     if self.peer_keyboard_enabled() {
@@ -1560,8 +1560,35 @@ impl Connection {
                         self.input_mouse(me, self.inner.id());
                     }
                 }
-                Some(message::Union::PointerDeviceEvent(pde)) =>
-                {
+                Some(message::Union::PointerDeviceEvent(pde)) => {
+                    #[cfg(any(target_os = "android", target_os = "ios"))]
+                    if let Err(e) = match pde.union {
+                        Some(pointer_device_event::Union::TouchEvent(touch)) => match touch.union {
+                            Some(touch_event::Union::PanStart(pan_start)) => {
+                                call_main_service_pointer_input(
+                                    "touch",
+                                    4,
+                                    pan_start.x,
+                                    pan_start.y,
+                                )
+                            }
+                            Some(touch_event::Union::PanUpdate(pan_update)) => {
+                                call_main_service_pointer_input(
+                                    "touch",
+                                    5,
+                                    pan_update.x,
+                                    pan_update.y,
+                                )
+                            }
+                            Some(touch_event::Union::PanEnd(pan_end)) => {
+                                call_main_service_pointer_input("touch", 6, pan_end.x, pan_end.y)
+                            }
+                            _ => Ok(()),
+                        },
+                        _ => Ok(()),
+                    } {
+                        log::debug!("call_main_service_pointer_input fail:{}", e);
+                    }
                     #[cfg(not(any(target_os = "android", target_os = "ios")))]
                     if self.peer_keyboard_enabled() {
                         MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);

src/ui_session_interface.rs

@@ -724,6 +724,49 @@ impl<T: InvokeUiSession> Session<T> {
         send_pointer_device_event(evt, alt, ctrl, shift, command, self);
     }
 
+    pub fn send_touch_pan_event(
+        &self,
+        event: &str,
+        x: i32,
+        y: i32,
+        alt: bool,
+        ctrl: bool,
+        shift: bool,
+        command: bool,
+    ) {
+        let mut touch_evt = TouchEvent::new();
+        match event {
+            "pan_start" => {
+                touch_evt.set_pan_start(TouchPanStart {
+                    x,
+                    y,
+                    ..Default::default()
+                });
+            }
+            "pan_update" => {
+                touch_evt.set_pan_update(TouchPanUpdate {
+                    x,
+                    y,
+                    ..Default::default()
+                });
+            }
+            "pan_end" => {
+                touch_evt.set_pan_end(TouchPanEnd {
+                    x,
+                    y,
+                    ..Default::default()
+                });
+            }
+            _ => {
+                log::warn!("unknown touch pan event: {}", event);
+                return;
+            }
+        };
+        let mut evt = PointerDeviceEvent::new();
+        evt.set_touch_event(touch_evt);
+        send_pointer_device_event(evt, alt, ctrl, shift, command, self);
+    }
+
     pub fn send_mouse(
         &self,
         mask: i32,