Compare commits

...

6 commits

Author SHA1 Message Date
Romain Vimont 907e55a66b WIP ignore duplicate mouse events 2019-09-22 21:42:43 +02:00
Romain Vimont 6d98ab5d29 WIP process touch events on client 2019-09-22 21:42:43 +02:00
Romain Vimont 91ba44b653 WIP handle touch events on server 2019-09-22 21:42:42 +02:00
Romain Vimont 7b143b8fbd WIP Add "inject touch" event 2019-09-22 21:38:17 +02:00
Romain Vimont 8d1f58278e Rename "pointer" to "mouse pointer"
This will help to distinguish them from "touch pointers".
2019-09-22 21:38:14 +02:00
Romain Vimont b2d815bfc7 Add buffer_write64be()
Add a function to write 64 bits in big-endian from a uint64_t.
2019-09-22 21:35:10 +02:00
15 changed files with 391 additions and 20 deletions

View file

@ -18,6 +18,12 @@ buffer_write32be(uint8_t *buf, uint32_t value) {
buf[3] = value;
}
static inline void
buffer_write64be(uint8_t *buf, uint64_t value) {
buffer_write32be(buf, value >> 32);
buffer_write32be(&buf[4], (uint32_t) value);
}
static inline uint16_t
buffer_read16be(const uint8_t *buf) {
return (buf[0] << 8) | buf[1];

View file

@ -1,6 +1,7 @@
#include "control_msg.h"
#include <string.h>
#include <SDL_assert.h>
#include "buffer_util.h"
#include "log.h"
@ -23,6 +24,16 @@ write_string(const char *utf8, size_t max_len, unsigned char *buf) {
return 2 + len;
}
static uint16_t
to_fixed_point_16(float f) {
SDL_assert(f >= 0.0f && f <= 1.0f);
uint32_t u = f * 0x1p16f; // 2^16
if (u >= 0xffff) {
u = 0xffff;
}
return (uint16_t) u;
}
size_t
control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
buf[0] = msg->type;
@ -42,6 +53,14 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
buffer_write32be(&buf[2], msg->inject_mouse_event.buttons);
write_position(&buf[6], &msg->inject_mouse_event.position);
return 18;
case CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT:
buf[1] = msg->inject_touch_event.action;
buffer_write64be(&buf[2], msg->inject_touch_event.finger_id);
write_position(&buf[10], &msg->inject_touch_event.position);
uint16_t pressure =
to_fixed_point_16(msg->inject_touch_event.pressure);
buffer_write16be(&buf[22], pressure);
return 24;
case CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT:
write_position(&buf[1], &msg->inject_scroll_event.position);
buffer_write32be(&buf[13],

View file

@ -18,6 +18,7 @@ enum control_msg_type {
CONTROL_MSG_TYPE_INJECT_KEYCODE,
CONTROL_MSG_TYPE_INJECT_TEXT,
CONTROL_MSG_TYPE_INJECT_MOUSE_EVENT,
CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT,
CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON,
CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL,
@ -49,6 +50,12 @@ struct control_msg {
enum android_motionevent_buttons buttons;
struct position position;
} inject_mouse_event;
struct {
enum android_motionevent_action action;
uint64_t finger_id;
struct position position;
float pressure;
} inject_touch_event;
struct {
struct position position;
int32_t hscroll;

View file

@ -205,6 +205,34 @@ convert_mouse_motion(const SDL_MouseMotionEvent *from, struct size screen_size,
return true;
}
static bool
convert_touch_action(SDL_EventType from, enum android_motionevent_action *to) {
switch (from) {
MAP(SDL_FINGERMOTION, AMOTION_EVENT_ACTION_MOVE);
MAP(SDL_FINGERDOWN, AMOTION_EVENT_ACTION_DOWN);
MAP(SDL_FINGERUP, AMOTION_EVENT_ACTION_UP);
FAIL;
}
}
bool
convert_touch(const SDL_TouchFingerEvent *from, struct size screen_size,
struct control_msg *to) {
to->type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
if (!convert_touch_action(from->type, &to->inject_touch_event.action)) {
return false;
}
to->inject_touch_event.finger_id = from->fingerId;
to->inject_touch_event.position.screen_size = screen_size;
// SDL touch event coordinates are normalized in the range [0; 1]
to->inject_touch_event.position.point.x = from->x * screen_size.width;
to->inject_touch_event.position.point.y = from->x * screen_size.height;
to->inject_touch_event.pressure = from->pressure;
return true;
}
bool
convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct position position,
struct control_msg *to) {

View file

@ -29,6 +29,10 @@ bool
convert_mouse_motion(const SDL_MouseMotionEvent *from, struct size screen_size,
struct control_msg *to);
bool
convert_touch(const SDL_TouchFingerEvent *from, struct size screen_size,
struct control_msg *to);
// on Android, a scroll event requires the current mouse position
bool
convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct position position,

View file

@ -387,6 +387,10 @@ input_manager_process_mouse_motion(struct input_manager *input_manager,
// do not send motion events when no button is pressed
return;
}
if (event->which == SDL_TOUCH_MOUSEID) {
// simulated from touch events, so it's a duplicate
return;
}
struct control_msg msg;
if (convert_mouse_motion(event, input_manager->screen->frame_size, &msg)) {
if (!controller_push_msg(input_manager->controller, &msg)) {
@ -395,6 +399,17 @@ input_manager_process_mouse_motion(struct input_manager *input_manager,
}
}
void
input_manager_process_touch(struct input_manager *input_manager,
const SDL_TouchFingerEvent *event) {
struct control_msg msg;
if (convert_touch(event, input_manager->screen->frame_size, &msg)) {
if (!controller_push_msg(input_manager->controller, &msg)) {
LOGW("Could not request 'inject touch event'");
}
}
}
static bool
is_outside_device_screen(struct input_manager *input_manager, int x, int y)
{

View file

@ -28,6 +28,10 @@ void
input_manager_process_mouse_motion(struct input_manager *input_manager,
const SDL_MouseMotionEvent *event);
void
input_manager_process_touch(struct input_manager *input_manager,
const SDL_TouchFingerEvent *event);
void
input_manager_process_mouse_button(struct input_manager *input_manager,
const SDL_MouseButtonEvent *event,

View file

@ -180,6 +180,11 @@ handle_event(SDL_Event *event, bool control) {
input_manager_process_mouse_button(&input_manager, &event->button,
control);
break;
case SDL_FINGERMOTION:
case SDL_FINGERDOWN:
case SDL_FINGERUP:
input_manager_process_touch(&input_manager, &event->tfinger);
break;
case SDL_DROPFILE: {
if (!control) {
break;

View file

@ -99,6 +99,41 @@ static void test_serialize_inject_mouse_event(void) {
};
assert(!memcmp(buf, expected, sizeof(expected)));
}
#include <stdio.h>
static void test_serialize_inject_touch_event(void) {
struct control_msg msg = {
.type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
.inject_touch_event = {
.action = AMOTION_EVENT_ACTION_DOWN,
.finger_id = 0x1234567887654321L,
.position = {
.point = {
.x = 100,
.y = 200,
},
.screen_size = {
.width = 1080,
.height = 1920,
},
},
.pressure = 1.0f,
},
};
unsigned char buf[CONTROL_MSG_SERIALIZED_MAX_SIZE];
int size = control_msg_serialize(&msg, buf);
assert(size == 24);
const unsigned char expected[] = {
CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
0x00, // AKEY_EVENT_ACTION_DOWN
0x12, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 0x21, // finger id
0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xc8, // 100 200
0x04, 0x38, 0x07, 0x80, // 1080 1920
0xff, 0xff, // pressure
};
assert(!memcmp(buf, expected, sizeof(expected)));
}
static void test_serialize_inject_scroll_event(void) {
struct control_msg msg = {
@ -237,6 +272,7 @@ int main(void) {
test_serialize_inject_text();
test_serialize_inject_text_long();
test_serialize_inject_mouse_event();
test_serialize_inject_touch_event();
test_serialize_inject_scroll_event();
test_serialize_back_or_screen_on();
test_serialize_expand_notification_panel();

View file

@ -8,13 +8,14 @@ public final class ControlMessage {
public static final int TYPE_INJECT_KEYCODE = 0;
public static final int TYPE_INJECT_TEXT = 1;
public static final int TYPE_INJECT_MOUSE_EVENT = 2;
public static final int TYPE_INJECT_SCROLL_EVENT = 3;
public static final int TYPE_BACK_OR_SCREEN_ON = 4;
public static final int TYPE_EXPAND_NOTIFICATION_PANEL = 5;
public static final int TYPE_COLLAPSE_NOTIFICATION_PANEL = 6;
public static final int TYPE_GET_CLIPBOARD = 7;
public static final int TYPE_SET_CLIPBOARD = 8;
public static final int TYPE_SET_SCREEN_POWER_MODE = 9;
public static final int TYPE_INJECT_TOUCH_EVENT = 3;
public static final int TYPE_INJECT_SCROLL_EVENT = 4;
public static final int TYPE_BACK_OR_SCREEN_ON = 5;
public static final int TYPE_EXPAND_NOTIFICATION_PANEL = 6;
public static final int TYPE_COLLAPSE_NOTIFICATION_PANEL = 7;
public static final int TYPE_GET_CLIPBOARD = 8;
public static final int TYPE_SET_CLIPBOARD = 9;
public static final int TYPE_SET_SCREEN_POWER_MODE = 10;
private int type;
private String text;
@ -22,6 +23,8 @@ public final class ControlMessage {
private int action; // KeyEvent.ACTION_* or MotionEvent.ACTION_* or POWER_MODE_*
private int keycode; // KeyEvent.KEYCODE_*
private int buttons; // MotionEvent.BUTTON_*
private long fingerId;
private float pressure;
private Position position;
private int hScroll;
private int vScroll;
@ -54,6 +57,16 @@ public final class ControlMessage {
return msg;
}
public static ControlMessage createInjectTouchEvent(int action, long fingerId, Position position, float pressure) {
ControlMessage msg = new ControlMessage();
msg.type = TYPE_INJECT_TOUCH_EVENT;
msg.action = action;
msg.fingerId = fingerId;
msg.pressure = pressure;
msg.position = position;
return msg;
}
public static ControlMessage createInjectScrollEvent(Position position, int hScroll, int vScroll) {
ControlMessage msg = new ControlMessage();
msg.type = TYPE_INJECT_SCROLL_EVENT;
@ -110,6 +123,14 @@ public final class ControlMessage {
return buttons;
}
public long getFingerId() {
return fingerId;
}
public float getPressure() {
return pressure;
}
public Position getPosition() {
return position;
}

View file

@ -10,6 +10,7 @@ public class ControlMessageReader {
private static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 9;
private static final int INJECT_MOUSE_EVENT_PAYLOAD_LENGTH = 17;
private static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 21;
private static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
private static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
@ -62,6 +63,9 @@ public class ControlMessageReader {
case ControlMessage.TYPE_INJECT_MOUSE_EVENT:
msg = parseInjectMouseEvent();
break;
case ControlMessage.TYPE_INJECT_TOUCH_EVENT:
msg = parseInjectTouchEvent();
break;
case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
msg = parseInjectScrollEvent();
break;
@ -130,6 +134,20 @@ public class ControlMessageReader {
return ControlMessage.createInjectMouseEvent(action, buttons, position);
}
private ControlMessage parseInjectTouchEvent() {
if (buffer.remaining() < INJECT_TOUCH_EVENT_PAYLOAD_LENGTH) {
return null;
}
int action = toUnsigned(buffer.get());
long fingerId = buffer.getLong();
Position position = readPosition(buffer);
// 16 bits fixed-point
int pressureInt = toUnsigned(buffer.getShort());
// convert it to a float between 0 and 1 (0x1p16f is 2^16 as float)
float pressure = pressureInt == 0xffff ? 1f : (pressureInt / 0x1p16f);
return ControlMessage.createInjectTouchEvent(action, fingerId, position, pressure);
}
private ControlMessage parseInjectScrollEvent() {
if (buffer.remaining() < INJECT_SCROLL_EVENT_PAYLOAD_LENGTH) {
return null;

View file

@ -13,6 +13,8 @@ import java.io.IOException;
public class Controller {
private static final int MAX_FINGERS = 10;
private final Device device;
private final DesktopConnection connection;
private final DeviceMessageSender sender;
@ -20,35 +22,52 @@ public class Controller {
private final KeyCharacterMap charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD);
private long lastMouseDown;
private final MotionEvent.PointerProperties[] pointerProperties = {new MotionEvent.PointerProperties()};
private final MotionEvent.PointerCoords[] pointerCoords = {new MotionEvent.PointerCoords()};
private final MotionEvent.PointerProperties[] mousePointerProperties = {new MotionEvent.PointerProperties()};
private final MotionEvent.PointerCoords[] mousePointerCoords = {new MotionEvent.PointerCoords()};
private long lastTouchDown;
private final FingersState fingersState = new FingersState(MAX_FINGERS);
private final MotionEvent.PointerProperties[] touchPointerProperties = new MotionEvent.PointerProperties[MAX_FINGERS];
private final MotionEvent.PointerCoords[] touchPointerCoords = new MotionEvent.PointerCoords[MAX_FINGERS];
public Controller(Device device, DesktopConnection connection) {
this.device = device;
this.connection = connection;
initPointer();
initMousePointer();
initTouchPointers();
sender = new DeviceMessageSender(connection);
}
private void initPointer() {
MotionEvent.PointerProperties props = pointerProperties[0];
private void initMousePointer() {
MotionEvent.PointerProperties props = mousePointerProperties[0];
props.id = 0;
props.toolType = MotionEvent.TOOL_TYPE_FINGER;
MotionEvent.PointerCoords coords = pointerCoords[0];
MotionEvent.PointerCoords coords = mousePointerCoords[0];
coords.orientation = 0;
coords.pressure = 1;
coords.size = 1;
}
private void setPointerCoords(Point point) {
MotionEvent.PointerCoords coords = pointerCoords[0];
private void initTouchPointers() {
for (int i = 0; i < MAX_FINGERS; ++i) {
MotionEvent.PointerProperties props = new MotionEvent.PointerProperties();
props.toolType = MotionEvent.TOOL_TYPE_FINGER;
MotionEvent.PointerCoords coords = new MotionEvent.PointerCoords();
coords.orientation = 0;
coords.size = 1;
}
}
private void setMousePointerCoords(Point point) {
MotionEvent.PointerCoords coords = mousePointerCoords[0];
coords.x = point.getX();
coords.y = point.getY();
}
private void setScroll(int hScroll, int vScroll) {
MotionEvent.PointerCoords coords = pointerCoords[0];
MotionEvent.PointerCoords coords = mousePointerCoords[0];
coords.setAxisValue(MotionEvent.AXIS_HSCROLL, hScroll);
coords.setAxisValue(MotionEvent.AXIS_VSCROLL, vScroll);
}
@ -158,8 +177,36 @@ public class Controller {
// ignore event
return false;
}
setPointerCoords(point);
MotionEvent event = MotionEvent.obtain(lastMouseDown, now, action, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, 0, 0,
setMousePointerCoords(point);
MotionEvent event = MotionEvent.obtain(lastMouseDown, now, action, 1, mousePointerProperties, mousePointerCoords, 0, buttons, 1f, 1f, 0, 0,
InputDevice.SOURCE_TOUCHSCREEN, 0);
return injectEvent(event);
}
private boolean injectTouch(int action, int fingerId, Position position, float pressure) {
long now = SystemClock.uptimeMillis();
if (action == MotionEvent.ACTION_DOWN) {
lastTouchDown = now;
}
Point point = device.getPhysicalPoint(position);
if (point == null) {
// ignore event
return false;
}
if (action == MotionEvent.ACTION_UP) {
if (!fingersState.unset(fingerId)) {
Ln.w("Unexpected ACTION_UP on unknown finger");
return false;
}
} else {
// ACTION_DOWN or ACTION_MOVE
if (!fingersState.set(fingerId, point, pressure)) {
Ln.w("Too many fingers for touch event");
return false;
}
}
int pointerCount = fingersState.update(touchPointerProperties, touchPointerCoords);
MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, touchPointerProperties, touchPointerCoords, 0, 0, 1f, 1f, 0, 0,
InputDevice.SOURCE_TOUCHSCREEN, 0);
return injectEvent(event);
}
@ -171,9 +218,9 @@ public class Controller {
// ignore event
return false;
}
setPointerCoords(point);
setMousePointerCoords(point);
setScroll(hScroll, vScroll);
MotionEvent event = MotionEvent.obtain(lastMouseDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, 0,
MotionEvent event = MotionEvent.obtain(lastMouseDown, now, MotionEvent.ACTION_SCROLL, 1, mousePointerProperties, mousePointerCoords, 0, 0, 1f, 1f, 0,
0, InputDevice.SOURCE_MOUSE, 0);
return injectEvent(event);
}

View file

@ -0,0 +1,32 @@
package com.genymobile.scrcpy;
public class Finger {
private final long id;
private Point point;
private float pressure;
public Finger(long id) {
this.id = id;
}
public long getId() {
return id;
}
public Point getPoint() {
return point;
}
public void setPoint(Point point) {
this.point = point;
}
public float getPressure() {
return pressure;
}
public void setPressure(float pressure) {
this.pressure = pressure;
}
}

View file

@ -0,0 +1,99 @@
package com.genymobile.scrcpy;
import android.view.MotionEvent;
public class FingersState {
/**
* Array of enabled fingers (can contain "null" holes).
* <p>
* Once a Finger (identified by its id received from the client) is enabled, it is never moved.
* <p>
* Its index is its local identifier injected into MotionEvents.
*/
private final Finger[] fingers;
public FingersState(int maxFingers) {
fingers = new Finger[maxFingers];
}
private int indexOf(long id) {
for (int i = 0; i < fingers.length; ++i) {
Finger finger = fingers[i];
if (finger != null && finger.getId() == id) {
return i;
}
}
return -1;
}
private int indexOfFirstEmpty() {
for (int i = 0; i < fingers.length; ++i) {
if (fingers[i] == null) {
return i;
}
}
return -1;
}
private Finger create(long id) {
int index = indexOf(id);
if (index != -1) {
// already exists, return it
return fingers[index];
}
int firstEmpty = indexOfFirstEmpty();
if (firstEmpty == -1) {
// it's full
return null;
}
Finger finger = new Finger(id);
fingers[firstEmpty] = finger;
return finger;
}
public boolean unset(int id) {
int index = indexOf(id);
if (index == -1) {
return false;
}
fingers[index] = null;
return true;
}
public boolean set(int id, Point point, float pressure) {
Finger finger = create(id);
if (finger == null) {
return false;
}
finger.setPoint(point);
finger.setPressure(pressure);
return true;
}
/**
* Initialize the motion event parameters.
*
* @param props the pointer properties
* @param coords the pointer coordinates
* @return The number of items initialized (the number of fingers).
*/
public int update(MotionEvent.PointerProperties[] props, MotionEvent.PointerCoords[] coords) {
int count = 0;
for (int i = 0; i < fingers.length; ++i) {
Finger finger = fingers[i];
if (finger != null) {
// id 0 is reserved for mouse events
props[count].id = i + 1;
Point point = finger.getPoint();
coords[i].x = point.getX();
coords[i].y = point.getY();
coords[i].pressure = finger.getPressure();
++count;
}
}
return count;
}
}

View file

@ -104,6 +104,36 @@ public class ControlMessageReaderTest {
Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight());
}
@Test
public void testParseTouchEvent() throws IOException {
ControlMessageReader reader = new ControlMessageReader();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
dos.writeByte(ControlMessage.TYPE_INJECT_TOUCH_EVENT);
dos.writeByte(MotionEvent.ACTION_DOWN);
dos.writeLong(-42); // fingerId
dos.writeInt(100);
dos.writeInt(200);
dos.writeShort(1080);
dos.writeShort(1920);
dos.writeShort(0xffff); // pressure
byte[] packet = bos.toByteArray();
reader.readFrom(new ByteArrayInputStream(packet));
ControlMessage event = reader.next();
Assert.assertEquals(ControlMessage.TYPE_INJECT_TOUCH_EVENT, event.getType());
Assert.assertEquals(MotionEvent.ACTION_DOWN, event.getAction());
Assert.assertEquals(-42, event.getFingerId());
Assert.assertEquals(100, event.getPosition().getPoint().getX());
Assert.assertEquals(200, event.getPosition().getPoint().getY());
Assert.assertEquals(1080, event.getPosition().getScreenSize().getWidth());
Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight());
Assert.assertEquals(1f, event.getPressure(), 0f); // must be exact
}
@Test
@SuppressWarnings("checkstyle:MagicNumber")
public void testParseScrollEvent() throws IOException {