maxmustermann/scrcpy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Wait SET_CLIPBOARD ack before Ctrl+v via HID

Browse source 
To allow seamless copy-paste, on Ctrl+v, a SET_CLIPBOARD request is
performed before injecting Ctrl+v.

But when HID keyboard is enabled, the Ctrl+v injection is not sent on
the same channel as the clipboard request, so they are not serialized,
and may occur in any order. If Ctrl+v happens to be injected before the
new clipboard content is set, then the old content is pasted instead,
which is incorrect.

To minimize the probability of occurrence of the wrong order, a delay of
2 milliseconds was added before injecting Ctrl+v. Then 5ms. But even
with 5ms, the wrong behavior sometimes happens.

To handle it properly, add an acknowledgement mechanism, so that Ctrl+v
is injected over AOA only after the SET_CLIPBOARD request has been
performed and acknowledged by the server.

Refs e4163321f0
Refs 45b0f8123a

PR #2814 <https://github.com/Genymobile/scrcpy/pull/2814>
This commit is contained in:
Romain Vimont 2021-11-21 17:40:11 +01:00
parent 2d5525eac1
commit 5d17bcf1bc
13 changed files with 135 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
app/src/aoa_hid.c
View file

@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
hid_event->accessory_id = accessory_id;
hid_event->buffer = buffer;
hid_event->size = buffer_size;
hid_event->delay = 0;
hid_event->ack_to_wait = SC_SEQUENCE_INVALID;
}
void
@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
}
bool
sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
sc_aoa_init(struct sc_aoa *aoa, const char *serial,
struct sc_acksync *acksync) {
assert(acksync);
cbuf_init(&aoa->queue);
if (!sc_mutex_init(&aoa->mutex)) {
@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
}
aoa->stopped = false;
aoa->acksync = acksync;
return true;
}
@ -332,23 +336,28 @@ run_aoa_thread(void *data) {
assert(non_empty);
(void) non_empty;
assert(event.delay >= 0);
if (event.delay) {
// Wait during the specified delay before injecting the HID event
sc_tick deadline = sc_tick_now() + event.delay;
bool timed_out = false;
while (!aoa->stopped && !timed_out) {
timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex,
deadline);
}
if (aoa->stopped) {
sc_mutex_unlock(&aoa->mutex);
uint64_t ack_to_wait = event.ack_to_wait;
sc_mutex_unlock(&aoa->mutex);
if (ack_to_wait != SC_SEQUENCE_INVALID) {
LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait);
// Do not block the loop indefinitely if the ack never comes (it should
// never happen)
sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500);
enum sc_acksync_wait_result result =
sc_acksync_wait(aoa->acksync, ack_to_wait, deadline);
if (result == SC_ACKSYNC_WAIT_TIMEOUT) {
LOGW("Ack not received after 500ms, discarding HID event");
sc_hid_event_destroy(&event);
continue;
} else if (result == SC_ACKSYNC_WAIT_INTR) {
// stopped
sc_hid_event_destroy(&event);
break;
}
}
sc_mutex_unlock(&aoa->mutex);
bool ok = sc_aoa_send_hid_event(aoa, &event);
sc_hid_event_destroy(&event);
if (!ok) {
@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) {
aoa->stopped = true;
sc_cond_signal(&aoa->event_cond);
sc_mutex_unlock(&aoa->mutex);
sc_acksync_interrupt(aoa->acksync);
}
void

7
app/src/aoa_hid.h
View file

@ -6,6 +6,7 @@
#include <libusb-1.0/libusb.h>
#include "util/acksync.h"
#include "util/cbuf.h"
#include "util/thread.h"
#include "util/tick.h"
@ -14,7 +15,7 @@ struct sc_hid_event {
uint16_t accessory_id;
unsigned char *buffer;
uint16_t size;
sc_tick delay;
uint64_t ack_to_wait;
};
// Takes ownership of buffer
@ -36,10 +37,12 @@ struct sc_aoa {
sc_cond event_cond;
bool stopped;
struct sc_hid_event_queue queue;
struct sc_acksync *acksync;
};
bool
sc_aoa_init(struct sc_aoa *aoa, const char *serial);
sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync);
void
sc_aoa_destroy(struct sc_aoa *aoa);

5
app/src/controller.c
View file

@ -5,10 +5,11 @@
#include "util/log.h"
bool
controller_init(struct controller *controller, sc_socket control_socket) {
controller_init(struct controller *controller, sc_socket control_socket,
struct sc_acksync *acksync) {
cbuf_init(&controller->queue);
bool ok = receiver_init(&controller->receiver, control_socket);
bool ok = receiver_init(&controller->receiver, control_socket, acksync);
if (!ok) {
return false;
}

4
app/src/controller.h
View file

@ -7,6 +7,7 @@
#include "control_msg.h"
#include "receiver.h"
#include "util/acksync.h"
#include "util/cbuf.h"
#include "util/net.h"
#include "util/thread.h"
@ -24,7 +25,8 @@ struct controller {
};
bool
controller_init(struct controller *controller, sc_socket control_socket);
controller_init(struct controller *controller, sc_socket control_socket,
struct sc_acksync *acksync);
void
controller_destroy(struct controller *controller);

16
app/src/hid_keyboard.c
View file

@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) {
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const SDL_KeyboardEvent *event,
bool device_clipboard_set) {
uint64_t ack_to_wait) {
if (event->repeat) {
// In USB HID protocol, key repeat is handled by the host (Android), so
// just ignore key repeat here.
@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
}
}
if (device_clipboard_set) {
if (ack_to_wait) {
// Ctrl+v is pressed, so clipboard synchronization has been
// requested. Wait a bit so that the clipboard is set before
// injecting Ctrl+v via HID, otherwise it would paste the old
// clipboard content.
hid_event.delay = SC_TICK_FROM_MS(5);
// requested. Wait until clipboard synchronization is acknowledged
// by the server, otherwise it could paste the old clipboard
// content.
hid_event.ack_to_wait = ack_to_wait;
}
if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
.process_text = sc_key_processor_process_text,
};
// Clipboard synchronization is requested over the control socket, while HID
// events are sent over AOA, so it must wait for clipboard synchronization
// to be acknowledged by the device before injecting Ctrl+v.
kb->key_processor.async_paste = true;
kb->key_processor.ops = &ops;
return true;

30
app/src/input_manager.c
View file

@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller,
im->last_keycode = SDLK_UNKNOWN;
im->last_mod = 0;
im->key_repeat = 0;
im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID
}
static void
@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) {
}
static bool
set_device_clipboard(struct controller *controller, bool paste) {
set_device_clipboard(struct controller *controller, bool paste,
uint64_t sequence) {
char *text = SDL_GetClipboardText();
if (!text) {
LOGW("Could not get clipboard text: %s", SDL_GetError());
@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) {
struct control_msg msg;
msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD;
msg.set_clipboard.sequence = 0; // unused for now
msg.set_clipboard.sequence = sequence;
msg.set_clipboard.text = text_dup;
msg.set_clipboard.paste = paste;
@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im,
// inject the text as input events
clipboard_paste(controller);
} else {
// store the text in the device clipboard and paste
set_device_clipboard(controller, true);
// store the text in the device clipboard and paste,
// without requesting an acknowledgment
set_device_clipboard(controller, true,
SC_SEQUENCE_INVALID);
}
}
return;
@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im,
return;
}
uint64_t ack_to_wait = SC_SEQUENCE_INVALID;
bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat;
if (is_ctrl_v) {
if (im->legacy_paste) {
@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im,
clipboard_paste(controller);
return;
}
// Request an acknowledgement only if necessary
uint64_t sequence = im->kp->async_paste ? im->next_sequence
: SC_SEQUENCE_INVALID;
// Synchronize the computer clipboard to the device clipboard before
// sending Ctrl+v, to allow seamless copy-paste.
bool ok = set_device_clipboard(controller, false);
bool ok = set_device_clipboard(controller, false, sequence);
if (!ok) {
LOGW("Clipboard could not be synchronized, Ctrl+v not injected");
return;
}
if (im->kp->async_paste) {
// The key processor must wait for this ack before injecting Ctrl+v
ack_to_wait = sequence;
// Increment only when the request succeeded
++im->next_sequence;
}
}
im->kp->ops->process_key(im->kp, event, is_ctrl_v);
im->kp->ops->process_key(im->kp, event, ack_to_wait);
}
static void

2
app/src/input_manager.h
View file

@ -38,6 +38,8 @@ struct input_manager {
unsigned key_repeat;
SDL_Keycode last_keycode;
uint16_t last_mod;
uint64_t next_sequence; // used for request acknowledgements
};
void

6
app/src/keyboard_inject.c
View file

@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to,
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const SDL_KeyboardEvent *event,
bool device_clipboard_set) {
uint64_t ack_to_wait) {
// The device clipboard synchronization and the key event messages are
// serialized, there is nothing special to do to ensure that the clipboard
// is set before injecting Ctrl+v.
(void) device_clipboard_set;
(void) ack_to_wait;
struct sc_keyboard_inject *ki = DOWNCAST(kp);
@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki,
.process_text = sc_key_processor_process_text,
};
// Key injection and clipboard synchronization are serialized
ki->key_processor.async_paste = false;
ki->key_processor.ops = &ops;
}

19
app/src/receiver.c
View file

@ -7,12 +7,16 @@
#include "util/log.h"
bool
receiver_init(struct receiver *receiver, sc_socket control_socket) {
receiver_init(struct receiver *receiver, sc_socket control_socket,
struct sc_acksync *acksync) {
bool ok = sc_mutex_init(&receiver->mutex);
if (!ok) {
return false;
}
receiver->control_socket = control_socket;
receiver->acksync = acksync;
return true;
}
@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) {
}
static void
process_msg(struct device_msg *msg) {
process_msg(struct receiver *receiver, struct device_msg *msg) {
switch (msg->type) {
case DEVICE_MSG_TYPE_CLIPBOARD: {
char *current = SDL_GetClipboardText();
@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) {
break;
}
case DEVICE_MSG_TYPE_ACK_CLIPBOARD:
// TODO
assert(receiver->acksync);
LOGD("Ack device clipboard sequence=%" PRIu64_,
msg->ack_clipboard.sequence);
sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
break;
}
}
static ssize_t
process_msgs(const unsigned char *buf, size_t len) {
process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) {
size_t head = 0;
for (;;) {
struct device_msg msg;
@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) {
return head;
}
process_msg(&msg);
process_msg(receiver, &msg);
device_msg_destroy(&msg);
head += r;
@ -84,7 +91,7 @@ run_receiver(void *data) {
}
head += r;
ssize_t consumed = process_msgs(buf, head);
ssize_t consumed = process_msgs(receiver, buf, head);
if (consumed == -1) {
// an error occurred
break;

6
app/src/receiver.h
View file

@ -5,6 +5,7 @@
#include <stdbool.h>
#include "util/acksync.h"
#include "util/net.h"
#include "util/thread.h"
@ -14,10 +15,13 @@ struct receiver {
sc_socket control_socket;
sc_thread thread;
sc_mutex mutex;
struct sc_acksync *acksync;
};
bool
receiver_init(struct receiver *receiver, sc_socket control_socket);
receiver_init(struct receiver *receiver, sc_socket control_socket,
struct sc_acksync *acksync);
void
receiver_destroy(struct receiver *receiver);

23
app/src/scrcpy.c
View file

@ -27,6 +27,7 @@
#include "screen.h"
#include "server.h"
#include "stream.h"
#include "util/acksync.h"
#include "util/log.h"
#include "util/net.h"
#ifdef HAVE_V4L2
@ -46,6 +47,8 @@ struct scrcpy {
struct file_handler file_handler;
#ifdef HAVE_AOA_HID
struct sc_aoa aoa;
// sequence/ack helper to synchronize clipboard and Ctrl+v via HID
struct sc_acksync acksync;
#endif
union {
struct sc_keyboard_inject keyboard_inject;
@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) {
bool controller_started = false;
bool screen_initialized = false;
struct sc_acksync *acksync = NULL;
struct sc_server_params params = {
.serial = options->serial,
.log_level = options->log_level,
@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) {
}
if (options->control) {
if (!controller_init(&s->controller, s->server.control_socket)) {
#ifdef HAVE_AOA_HID
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
bool ok = sc_acksync_init(&s->acksync);
if (!ok) {
goto end;
}
acksync = &s->acksync;
}
#endif
if (!controller_init(&s->controller, s->server.control_socket,
acksync)) {
goto end;
}
controller_initialized = true;
@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) {
#ifdef HAVE_AOA_HID
bool aoa_hid_ok = false;
bool ok = sc_aoa_init(&s->aoa, serial);
bool ok = sc_aoa_init(&s->aoa, serial, acksync);
if (!ok) {
goto aoa_hid_end;
}
@ -586,6 +602,9 @@ end:
sc_hid_keyboard_destroy(&s->keyboard_hid);
sc_aoa_stop(&s->aoa);
}
if (acksync) {
sc_acksync_destroy(acksync);
}
#endif
if (controller_started) {
controller_stop(&s->controller);

18
app/src/trait/key_processor.h
View file

@ -14,6 +14,15 @@
* Component able to process and inject keys should implement this trait.
*/
struct sc_key_processor {
/**
* Set by the implementation to indicate that it must explicitly wait for
* the clipboard to be set on the device before injecting Ctrl+v to avoid
* race conditions. If it is set, the input_manager will pass a valid
* ack_to_wait to process_key() in case of clipboard synchronization
* resulting of the key event.
*/
bool async_paste;
const struct sc_key_processor_ops *ops;
};
@ -22,13 +31,14 @@ struct sc_key_processor_ops {
/**
* Process the keyboard event
*
* The flag `device_clipboard_set` indicates that the input manager sent a
* control message to synchronize the device clipboard as a result of this
* key event.
* The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates
* the acknowledgement number to wait for before injecting this event.
* This allows to ensure that the device clipboard is set before injecting
* Ctrl+v on the device.
*/
void
(*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event,
bool device_clipboard_set);
uint64_t ack_to_wait);
void
(*process_text)(struct sc_key_processor *kp,

5
app/src/util/acksync.h
View file

@ -9,6 +9,11 @@
/**
* Helper to wait for acknowledgments
*
* In practice, it is used to wait for device clipboard acknowledgement from the
* server before injecting Ctrl+v via AOA HID, in order to avoid pasting the
* content of the old device clipboard (if Ctrl+v was injected before the
* clipboard content was actually set).
*/
struct sc_acksync {
sc_mutex mutex;