From cf6452b7132c01635ec4e40ddf2d8da91c6dbb7d Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Sun, 5 Feb 2023 01:59:25 -0500 Subject: [PATCH 1/7] convert all times from micro to nano secs --- lib/byteutils.c | 16 +++++++----- lib/raop_ntp.c | 39 ++++++++++++++-------------- lib/raop_ntp.h | 2 +- lib/raop_rtp.c | 17 ++++++------ lib/raop_rtp_mirror.c | 7 ++--- renderers/audio_renderer_gstreamer.c | 6 +++-- renderers/video_renderer_gstreamer.c | 6 +++-- 7 files changed, 51 insertions(+), 42 deletions(-) diff --git a/lib/byteutils.c b/lib/byteutils.c index 4a5bdea..4798922 100644 --- a/lib/byteutils.c +++ b/lib/byteutils.c @@ -12,6 +12,8 @@ * Lesser General Public License for more details. */ +#define SECOND_IN_NSECS 1000000000UL + #include #ifdef _WIN32 # include @@ -96,23 +98,23 @@ void byteutils_put_int(unsigned char* b, int offset, uint32_t value) { } /** - * Reads an ntp timestamp and returns it as micro seconds since the Unix epoch + * Reads an ntp timestamp and returns it as nano seconds since the Unix epoch */ uint64_t byteutils_get_ntp_timestamp(unsigned char *b, int offset) { uint64_t seconds = ntohl(((unsigned int) byteutils_get_int(b, offset))) - SECONDS_FROM_1900_TO_1970; uint64_t fraction = ntohl((unsigned int) byteutils_get_int(b, offset + 4)); - return (seconds * 1000000L) + ((fraction * 1000000L) >> 32); + return (seconds * SECOND_IN_NSECS) + ((fraction * SECOND_IN_NSECS) >> 32); } /** - * Writes a time given as micro seconds since the Unix time epoch as an ntp timestamp + * Writes a time given as nano seconds since the Unix time epoch as an ntp timestamp * into the buffer at position offset */ -void byteutils_put_ntp_timestamp(unsigned char *b, int offset, uint64_t us_since_1970) { - uint64_t seconds = us_since_1970 / 1000000L; - uint64_t microseconds = us_since_1970 % 1000000L; +void byteutils_put_ntp_timestamp(unsigned char *b, int offset, uint64_t ns_since_1970) { + uint64_t seconds = ns_since_1970 / SECOND_IN_NSECS; + uint64_t nanoseconds = ns_since_1970 % SECOND_IN_NSECS; seconds += SECONDS_FROM_1900_TO_1970; - uint64_t fraction = (microseconds << 32) / 1000000L; + uint64_t fraction = (nanoseconds << 32) / SECOND_IN_NSECS; // Write in big endian! byteutils_put_int(b, offset, htonl(seconds)); diff --git a/lib/raop_ntp.c b/lib/raop_ntp.c index 7d79284..6318b82 100644 --- a/lib/raop_ntp.c +++ b/lib/raop_ntp.c @@ -33,6 +33,7 @@ #include "byteutils.h" #include "utils.h" +#define SECOND_IN_NSECS 1000000000UL #define RAOP_NTP_DATA_COUNT 8 #define RAOP_NTP_PHI_PPM 15ull // PPM #define RAOP_NTP_R_RHO ((1ull << 32) / 1000u) // packet precision @@ -284,7 +285,8 @@ raop_ntp_thread(void *arg) byteutils_put_ntp_timestamp(request, 24, send_time); int send_len = sendto(raop_ntp->tsock, (char *)request, sizeof(request), 0, (struct sockaddr *) &raop_ntp->remote_saddr, raop_ntp->remote_saddr_len); - logger_log(raop_ntp->logger, LOGGER_DEBUG, "\nraop_ntp send_len = %d, now = %llu", send_len, send_time); + logger_log(raop_ntp->logger, LOGGER_DEBUG, "\nraop_ntp send_len = %d, now = %8.6f", send_len, + (double) send_time / SECOND_IN_NSECS); if (send_len < 0) { logger_log(raop_ntp->logger, LOGGER_ERR, "raop_ntp error sending request"); } else { @@ -305,7 +307,6 @@ raop_ntp_thread(void *arg) } else { //local time of the server when the NTP response packet returns int64_t t3 = (int64_t) raop_ntp_get_local_time(raop_ntp); - timeout_counter = 0; char *str = utils_data_to_string(response, response_len, 16); logger_log(raop_ntp->logger, LOGGER_DEBUG, "raop_ntp receive time type_t=%d packetlen = %d\n%s", @@ -321,15 +322,15 @@ raop_ntp_thread(void *arg) // Local time of the client when the response message leaves the client int64_t t2 = (int64_t) byteutils_get_ntp_timestamp(response, 24); - // The iOS client device sends its time in micro seconds relative to an arbitrary Epoch (the last boot). - // For a little bonus confusion, they add SECONDS_FROM_1900_TO_1970 * 1000000 us. + // The iOS client device sends its time in seconds relative to an arbitrary Epoch (the last boot). + // For a little bonus confusion, they add SECONDS_FROM_1900_TO_1970. // This means we have to expect some rather huge offset, but its growth or shrink over time should be small. raop_ntp->data_index = (raop_ntp->data_index + 1) % RAOP_NTP_DATA_COUNT; raop_ntp->data[raop_ntp->data_index].time = t3; raop_ntp->data[raop_ntp->data_index].offset = ((t1 - t0) + (t2 - t3)) / 2; raop_ntp->data[raop_ntp->data_index].delay = ((t3 - t0) - (t2 - t1)); - raop_ntp->data[raop_ntp->data_index].dispersion = RAOP_NTP_R_RHO + RAOP_NTP_S_RHO + (t3 - t0) * RAOP_NTP_PHI_PPM / 1000000u; + raop_ntp->data[raop_ntp->data_index].dispersion = RAOP_NTP_R_RHO + RAOP_NTP_S_RHO + (t3 - t0) * RAOP_NTP_PHI_PPM / SECOND_IN_NSECS; // Sort by delay memcpy(data_sorted, raop_ntp->data, sizeof(data_sorted)); @@ -341,7 +342,7 @@ raop_ntp_thread(void *arg) // Calculate dispersion for(int i = 0; i < RAOP_NTP_DATA_COUNT; ++i) { - unsigned long long disp = raop_ntp->data[i].dispersion + (t3 - raop_ntp->data[i].time) * RAOP_NTP_PHI_PPM / 1000000u; + unsigned long long disp = raop_ntp->data[i].dispersion + (t3 - raop_ntp->data[i].time) * RAOP_NTP_PHI_PPM / SECOND_IN_NSECS; dispersion += disp / two_pow_n[i]; } @@ -454,54 +455,54 @@ raop_ntp_stop(raop_ntp_t *raop_ntp) } /** - * Converts from a little endian ntp timestamp to micro seconds since the Unix epoch. + * Converts from a little endian ntp timestamp to nano seconds since the Unix epoch. * Does the same thing as byteutils_get_ntp_timestamp, except its input is an uint64_t * and expected to already be in little endian. * Please note this just converts to a different representation, the clock remains the * same. */ -uint64_t raop_ntp_timestamp_to_micro_seconds(uint64_t ntp_timestamp, bool account_for_epoch_diff) { +uint64_t raop_ntp_timestamp_to_nano_seconds(uint64_t ntp_timestamp, bool account_for_epoch_diff) { uint64_t seconds = ((ntp_timestamp >> 32) & 0xffffffff) - (account_for_epoch_diff ? SECONDS_FROM_1900_TO_1970 : 0); uint64_t fraction = (ntp_timestamp & 0xffffffff); - return (seconds * 1000000) + ((fraction * 1000000) >> 32); + return (seconds * SECOND_IN_NSECS) + ((fraction * SECOND_IN_NSECS) >> 32); } /** - * Returns the current time in micro seconds according to the local wall clock. + * Returns the current time in nano seconds according to the local wall clock. * The system Unix time is used as the local wall clock. */ uint64_t raop_ntp_get_local_time(raop_ntp_t *raop_ntp) { struct timespec time; clock_gettime(CLOCK_REALTIME, &time); - return (uint64_t)time.tv_sec * 1000000L + (uint64_t)(time.tv_nsec / 1000); + return ((uint64_t)time.tv_sec) * SECOND_IN_NSECS + (uint64_t)(time.tv_nsec); } /** - * Returns the current time in micro seconds according to the remote wall clock. + * Returns the current time in nano seconds according to the remote wall clock. */ uint64_t raop_ntp_get_remote_time(raop_ntp_t *raop_ntp) { MUTEX_LOCK(raop_ntp->sync_params_mutex); int64_t offset = raop_ntp->sync_offset; MUTEX_UNLOCK(raop_ntp->sync_params_mutex); - return (uint64_t) ((int64_t) raop_ntp_get_local_time(raop_ntp)) + ((int64_t) offset); + return (uint64_t) (((int64_t) raop_ntp_get_local_time(raop_ntp)) + offset); } /** - * Returns the local wall clock time in micro seconds for the given point in remote clock time + * Returns the local wall clock time in nano seconds for the given point in remote clock time */ uint64_t raop_ntp_convert_remote_time(raop_ntp_t *raop_ntp, uint64_t remote_time) { MUTEX_LOCK(raop_ntp->sync_params_mutex); - uint64_t offset = raop_ntp->sync_offset; + int64_t offset = raop_ntp->sync_offset; MUTEX_UNLOCK(raop_ntp->sync_params_mutex); - return (uint64_t) ((int64_t) remote_time) - ((int64_t) offset); + return (uint64_t) (((int64_t) remote_time) - offset); } /** - * Returns the remote wall clock time in micro seconds for the given point in local clock time + * Returns the remote wall clock time in nano seconds for the given point in local clock time */ uint64_t raop_ntp_convert_local_time(raop_ntp_t *raop_ntp, uint64_t local_time) { MUTEX_LOCK(raop_ntp->sync_params_mutex); - uint64_t offset = raop_ntp->sync_offset; + int64_t offset = raop_ntp->sync_offset; MUTEX_UNLOCK(raop_ntp->sync_params_mutex); - return (uint64_t) ((int64_t) local_time) + ((int64_t) offset); + return (uint64_t) (((int64_t) local_time) + offset); } diff --git a/lib/raop_ntp.h b/lib/raop_ntp.h index 9983ee8..cd95dda 100644 --- a/lib/raop_ntp.h +++ b/lib/raop_ntp.h @@ -31,7 +31,7 @@ unsigned short raop_ntp_get_port(raop_ntp_t *raop_ntp); void raop_ntp_destroy(raop_ntp_t *raop_rtp); -uint64_t raop_ntp_timestamp_to_micro_seconds(uint64_t ntp_timestamp, bool account_for_epoch_diff); +uint64_t raop_ntp_timestamp_to_nano_seconds(uint64_t ntp_timestamp, bool account_for_epoch_diff); uint64_t raop_ntp_get_local_time(raop_ntp_t *raop_ntp); uint64_t raop_ntp_get_remote_time(raop_ntp_t *raop_ntp); diff --git a/lib/raop_rtp.c b/lib/raop_rtp.c index 7c21526..f16c871 100644 --- a/lib/raop_rtp.c +++ b/lib/raop_rtp.c @@ -32,12 +32,13 @@ #define NO_FLUSH (-42) -#define RAOP_RTP_SAMPLE_RATE (44100.0 / 1000000.0) +#define SECOND_IN_NSECS 1000000000UL +#define RAOP_RTP_SAMPLE_RATE (44100.0 / SECOND_IN_NSECS) #define RAOP_RTP_SYNC_DATA_COUNT 8 -#define SEC 1000000 +#define SEC SECOND_IN_NSECS -#define DELAY_AAC 500000 //empirical, matches audio latency of about -0.5 sec after first clock sync event -#define DELAY_ALAC 2000000 //empirical, matches audio latency of about -2.0 sec after first clock sync event +#define DELAY_AAC 500000000 //empirical, matches audio latency of about -0.5 sec after first clock sync event +#define DELAY_ALAC 200000000 //empirical, matches audio latency of about -2.0 sec after first clock sync event /* note: it is unclear what will happen in the unlikely event that this code is running at the time of the unix-time * epoch event on 2038-01-19 at 3:14:08 UTC ! (but Apple will surely have removed AirPlay "legacy pairing" by then!) */ @@ -550,7 +551,7 @@ raop_rtp_thread_udp(void *arg) have_synced = true; } uint64_t sync_ntp_raw = byteutils_get_long_be(packet, 8); - uint64_t sync_ntp_remote = raop_ntp_timestamp_to_micro_seconds(sync_ntp_raw, true); + uint64_t sync_ntp_remote = raop_ntp_timestamp_to_nano_seconds(sync_ntp_raw, true); uint64_t sync_ntp_local = raop_ntp_convert_remote_time(raop_rtp->ntp, sync_ntp_remote); int64_t shift; switch (raop_rtp->ct) { @@ -631,12 +632,12 @@ raop_rtp_thread_udp(void *arg) offset_estimate_initialized = true; switch (raop_rtp->ct) { case 0x02: - delay = DELAY_ALAC; /* DELAY = 2000000 (2.0 sec) is empirical choice for ALAC */ + delay = DELAY_ALAC; /* DELAY = 2000000000 (2.0 sec) is empirical choice for ALAC */ logger_log(raop_rtp->logger, LOGGER_DEBUG, "Audio is ALAC: using initial latency estimate -%8.6f sec", ((double) delay) / SEC); break; case 0x08: - delay = DELAY_AAC; /* DELAY = 500000 (0.5 sec) is empirical choice for AAC-ELD */ + delay = DELAY_AAC; /* DELAY = 500000000 (0.5 sec) is empirical choice for AAC-ELD */ logger_log(raop_rtp->logger, LOGGER_DEBUG, "Audio is AAC: using initial latency estimate -%8.6f sec", ((double) delay ) / SEC); break; @@ -682,7 +683,7 @@ raop_rtp_thread_udp(void *arg) free(payload); uint64_t ntp_now = raop_ntp_get_local_time(raop_rtp->ntp); int64_t latency = ((int64_t) ntp_now) - ((int64_t) audio_data.ntp_time); - logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp audio: now = %8.6f, npt = %8.6f, latency = %8.6f, rtp_time=%u seqnum = %u", + logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp audio: now = %8.6f, ntp = %8.6f, latency = %8.6f, rtp_time=%u seqnum = %u", ((double) ntp_now ) / SEC, ((double) audio_data.ntp_time) / SEC, ((double) latency) / SEC, (uint32_t) rtp64_timestamp, seqnum); } diff --git a/lib/raop_rtp_mirror.c b/lib/raop_rtp_mirror.c index fcac4bc..93ac87f 100644 --- a/lib/raop_rtp_mirror.c +++ b/lib/raop_rtp_mirror.c @@ -48,7 +48,8 @@ #define CAST #endif -#define SEC 1000000 +#define SECOND_IN_NSECS 1000000000UL +#define SEC SECOND_IN_NSECS /* for MacOS, where SOL_TCP and TCP_KEEPIDLE are not defined */ #if !defined(SOL_TCP) && defined(IPPROTO_TCP) @@ -359,9 +360,9 @@ raop_rtp_mirror_thread(void *arg) // Conveniently, the video data is already stamped with the remote wall clock time, // so no additional clock syncing needed. The only thing odd here is that the video // ntp time stamps don't include the SECONDS_FROM_1900_TO_1970, so it's really just - // counting micro seconds since last boot. + // counting nano seconds since last boot. ntp_timestamp_raw = byteutils_get_long(packet, 8); - uint64_t ntp_timestamp_remote = raop_ntp_timestamp_to_micro_seconds(ntp_timestamp_raw, false); + uint64_t ntp_timestamp_remote = raop_ntp_timestamp_to_nano_seconds(ntp_timestamp_raw, false); uint64_t ntp_timestamp = raop_ntp_convert_remote_time(raop_rtp_mirror->ntp, ntp_timestamp_remote); uint64_t ntp_now = raop_ntp_get_local_time(raop_rtp_mirror->ntp); diff --git a/renderers/audio_renderer_gstreamer.c b/renderers/audio_renderer_gstreamer.c index 5a989bf..9df50bb 100644 --- a/renderers/audio_renderer_gstreamer.c +++ b/renderers/audio_renderer_gstreamer.c @@ -21,6 +21,7 @@ #include #include #include "audio_renderer.h" +#define SECOND_IN_NSECS 1000000000UL /* GStreamer Caps strings for Airplay-defined audio compression types (ct) */ @@ -201,11 +202,12 @@ void audio_renderer_start(unsigned char *ct) { void audio_renderer_render_buffer(unsigned char* data, int *data_len, unsigned short *seqnum, uint64_t *ntp_time) { GstBuffer *buffer; bool valid; - GstClockTime pts = (GstClockTime) (*ntp_time * 1000); /* convert from usec to nsec */ + GstClockTime pts = (GstClockTime) *ntp_time ; /* now in nsecs */ if (pts >= gst_audio_pipeline_base_time) { pts -= gst_audio_pipeline_base_time; } else { - logger_log(logger, LOGGER_ERR, "*** invalid *pts_raw < gst_audio_pipeline_base_time"); + logger_log(logger, LOGGER_ERR, "*** invalid ntp_time < gst_audio_pipeline_base_time\n%8.6f ntp_time\n%8.6f base_time", + ((double) *ntp_time) / SECOND_IN_NSECS, ((double) gst_audio_pipeline_base_time) / SECOND_IN_NSECS); return; } if (data_len == 0 || renderer == NULL) return; diff --git a/renderers/video_renderer_gstreamer.c b/renderers/video_renderer_gstreamer.c index b64e484..dd03a90 100644 --- a/renderers/video_renderer_gstreamer.c +++ b/renderers/video_renderer_gstreamer.c @@ -21,6 +21,7 @@ #include #include +#define SECOND_IN_NSECS 1000000000UL #ifdef X_DISPLAY_FIX #include #include "x_display_fix.h" @@ -220,11 +221,12 @@ void video_renderer_start() { void video_renderer_render_buffer(unsigned char* data, int *data_len, int *nal_count, uint64_t *ntp_time) { GstBuffer *buffer; - GstClockTime pts = (GstClockTime) (*ntp_time * 1000); /*convert from usec to nsec */ + GstClockTime pts = (GstClockTime) *ntp_time; /*now in nsecs */ if (pts >= gst_video_pipeline_base_time) { pts -= gst_video_pipeline_base_time; } else { - logger_log(logger, LOGGER_ERR, "*** invalid *pts_raw < gst_video_pipeline_base_time") ; + logger_log(logger, LOGGER_ERR, "*** invalid ntp_time < gst_video_pipeline_base_time\n%8.6f ntp_time\n%8.6f base_time", + ((double) *ntp_time) / SECOND_IN_NSECS, ((double) gst_video_pipeline_base_time) / SECOND_IN_NSECS); return; } g_assert(data_len != 0); From f6a18a5a37ad9501a02a2995eba0ab1c25c60d7d Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Sun, 5 Feb 2023 14:11:02 -0500 Subject: [PATCH 2/7] use rtp_clock_rate instead of rtp_sync_scale --- lib/raop_handlers.h | 3 ++- lib/raop_rtp.c | 23 ++++++++++------------- lib/raop_rtp.h | 4 ++-- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/raop_handlers.h b/lib/raop_handlers.h index ca5d1d1..759c72f 100644 --- a/lib/raop_handlers.h +++ b/lib/raop_handlers.h @@ -500,6 +500,7 @@ raop_handler_setup(raop_conn_t *conn, unsigned short cport = conn->raop->control_lport, dport = conn->raop->data_lport; unsigned short remote_cport = 0; unsigned char ct; + unsigned int sr = 44100; /* all AirPlay audio formats supported so far have sample rate 44.1kHz */ uint64_t uint_val = 0; plist_t req_stream_control_port_node = plist_dict_get_item(req_stream_node, "controlPort"); plist_get_uint_val(req_stream_control_port_node, &uint_val); @@ -544,7 +545,7 @@ raop_handler_setup(raop_conn_t *conn, } if (conn->raop_rtp) { - raop_rtp_start_audio(conn->raop_rtp, use_udp, remote_cport, &cport, &dport, ct); + raop_rtp_start_audio(conn->raop_rtp, use_udp, &remote_cport, &cport, &dport, &ct, &sr); logger_log(conn->raop->logger, LOGGER_DEBUG, "RAOP initialized success"); } else { logger_log(conn->raop->logger, LOGGER_ERR, "RAOP not initialized at SETUP, playing will fail!"); diff --git a/lib/raop_rtp.c b/lib/raop_rtp.c index f16c871..efed2bc 100644 --- a/lib/raop_rtp.c +++ b/lib/raop_rtp.c @@ -33,7 +33,6 @@ #define NO_FLUSH (-42) #define SECOND_IN_NSECS 1000000000UL -#define RAOP_RTP_SAMPLE_RATE (44100.0 / SECOND_IN_NSECS) #define RAOP_RTP_SYNC_DATA_COUNT 8 #define SEC SECOND_IN_NSECS @@ -54,7 +53,7 @@ struct raop_rtp_s { // Time and sync raop_ntp_t *ntp; - double rtp_sync_scale; + double rtp_clock_rate; int64_t rtp_sync_offset; raop_rtp_sync_data_t sync_data[RAOP_RTP_SYNC_DATA_COUNT]; int sync_data_index; @@ -164,7 +163,6 @@ raop_rtp_init(logger_t *logger, raop_callbacks_t *callbacks, raop_ntp_t *ntp, co raop_rtp->ntp = ntp; raop_rtp->rtp_sync_offset = 0; - raop_rtp->rtp_sync_scale = RAOP_RTP_SAMPLE_RATE; raop_rtp->sync_data_index = 0; for (int i = 0; i < RAOP_RTP_SYNC_DATA_COUNT; ++i) { raop_rtp->sync_data[i].ntp_time = 0; @@ -404,13 +402,13 @@ void raop_rtp_sync_clock(raop_rtp_t *raop_rtp, uint64_t ntp_time, uint64_t rtp_t for (int i = 0; i < RAOP_RTP_SYNC_DATA_COUNT; i++) { if (raop_rtp->sync_data[i].ntp_time == 0) continue; rtp_offset = ((int64_t) raop_rtp->sync_data[i].rtp_time) - ((int64_t) raop_rtp->sync_data[latest].rtp_time); - total_offsets += ((double) rtp_offset) / raop_rtp-> rtp_sync_scale; + total_offsets += ((double) rtp_offset) * raop_rtp-> rtp_clock_rate; total_offsets -= (double) (((int64_t) raop_rtp->sync_data[i].ntp_time) - ((int64_t) raop_rtp->sync_data[latest].ntp_time)); valid_data_count++; } total_offsets = (total_offsets / valid_data_count); rtp_offset = ((int64_t) raop_rtp->sync_data[latest].rtp_time) - ((int64_t) raop_rtp->rtp_start_time) + ((int64_t) shift); - total_offsets += ((double) rtp_offset) / raop_rtp->rtp_sync_scale; + total_offsets += ((double) rtp_offset) * raop_rtp->rtp_clock_rate; avg_offset = (int64_t) total_offsets; avg_offset -= ((int64_t) raop_rtp->sync_data[latest].ntp_time) - ((int64_t) raop_rtp->ntp_start_time); correction = avg_offset - raop_rtp->rtp_sync_offset; @@ -655,7 +653,7 @@ raop_rtp_thread_udp(void *arg) if (seqnum2 != seqnum) { /* for AAC-ELD only use copy 3 of the 3 copies of each frame */ rtp_count++; offset -= initial_offset; - sync_adjustment += ((double) offset) + (((double) sync_rtp) / raop_rtp->rtp_sync_scale); + sync_adjustment += ((double) offset) + (((double) sync_rtp) * raop_rtp->rtp_clock_rate); raop_rtp->rtp_sync_offset = initial_offset + (int64_t) (sync_adjustment / rtp_count); //logger_log(raop_rtp->logger, LOGGER_DEBUG, "initial estimate of rtp_sync_offset %d secnum = %u: %8.6f", // rtp_count, seqnum, ((double) raop_rtp->rtp_sync_offset) / SEC); @@ -671,7 +669,7 @@ raop_rtp_thread_udp(void *arg) unsigned short seqnum; uint64_t rtp64_timestamp; while ((payload = raop_buffer_dequeue(raop_rtp->buffer, &payload_size, &rtp64_timestamp, &seqnum, no_resend))) { - double elapsed_time = (((double) (rtp64_timestamp - (uint64_t) raop_rtp->rtp_start_time)) / raop_rtp->rtp_sync_scale); + double elapsed_time = (((double) (rtp64_timestamp - (uint64_t) raop_rtp->rtp_start_time)) * raop_rtp->rtp_clock_rate); audio_decode_struct audio_data; audio_data.data_len = payload_size; audio_data.data = payload; @@ -712,15 +710,13 @@ raop_rtp_thread_udp(void *arg) // Start rtp service, three udp ports void -raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short control_rport, - unsigned short *control_lport, unsigned short *data_lport, unsigned char ct) +raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, + unsigned short *control_lport, unsigned short *data_lport, unsigned char *ct, unsigned int *sr) { logger_log(raop_rtp->logger, LOGGER_INFO, "raop_rtp starting audio"); int use_ipv6 = 0; assert(raop_rtp); - assert(control_lport); - assert(data_lport); MUTEX_LOCK(raop_rtp->run_mutex); if (raop_rtp->running || !raop_rtp->joined) { @@ -728,12 +724,13 @@ raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short control_r return; } - raop_rtp->ct = ct; + raop_rtp->ct = *ct; + raop_rtp->rtp_clock_rate = SECOND_IN_NSECS / *sr; /* Initialize ports and sockets */ raop_rtp->control_lport = *control_lport; raop_rtp->data_lport = *data_lport; - raop_rtp->control_rport = control_rport; + raop_rtp->control_rport = *control_rport; if (raop_rtp->remote_saddr.ss_family == AF_INET6) { use_ipv6 = 1; } diff --git a/lib/raop_rtp.h b/lib/raop_rtp.h index 9950c0b..a1ca7a3 100644 --- a/lib/raop_rtp.h +++ b/lib/raop_rtp.h @@ -29,8 +29,8 @@ typedef struct raop_rtp_s raop_rtp_t; raop_rtp_t *raop_rtp_init(logger_t *logger, raop_callbacks_t *callbacks, raop_ntp_t *ntp, const unsigned char *remote, int remotelen, const unsigned char *aeskey, const unsigned char *aesiv); -void raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short control_rport, - unsigned short *control_lport, unsigned short *data_lport, unsigned char ct); +void raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, + unsigned short *control_lport, unsigned short *data_lport, unsigned char *ct, unsigned int *sr); void raop_rtp_set_volume(raop_rtp_t *raop_rtp, float volume); void raop_rtp_set_metadata(raop_rtp_t *raop_rtp, const char *data, int datalen); From c8025be7118d45850ecba2161f8129939bd84909 Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Mon, 6 Feb 2023 14:15:52 -0500 Subject: [PATCH 3/7] v1.63: fix audio sync with client video in Audio-only mode --- README.html | 42 ++++++++++++---------------- README.md | 25 ++++++++--------- README.txt | 39 +++++++++++--------------- lib/raop.c | 7 +++++ lib/raop_handlers.h | 13 +++++++-- lib/raop_rtp.c | 14 +++++----- lib/raop_rtp.h | 4 +-- renderers/audio_renderer.h | 2 +- renderers/audio_renderer_gstreamer.c | 16 ++++++----- uxplay.1 | 6 ++-- uxplay.cpp | 28 ++++++++----------- 11 files changed, 98 insertions(+), 98 deletions(-) diff --git a/README.html b/README.html index cfb1616..d588998 100644 --- a/README.html +++ b/README.html @@ -1,6 +1,6 @@

UxPlay -1.62: AirPlay-Mirror and AirPlay-Audio server for Linux, macOS, and Unix +id="uxplay-1.63-airplay-mirror-and-airplay-audio-server-for-linux-macos-and-unix-now-also-runs-on-windows.">UxPlay +1.63: AirPlay-Mirror and AirPlay-Audio server for Linux, macOS, and Unix (now also runs on Windows).

Now @@ -400,17 +400,11 @@ from sources like Apple Music in Audio-Only (ALAC) mode: run a image viewer with an autoreload feature: an example is “feh”: run “feh -R 1 <name>” in the foreground; terminate feh and then Uxplay with “ctrl-C fg ctrl-C”.

-

  • If you wish to listen in Audio-Only mode on the server while -watching the client screen (for video or Apple Music song lyrics, etc.), -the video on the client is delayed by about 5 seconds behind the the -audio on the server (this is a Legacy mode issue: the client does -not receive latency information to sync its video with audio played on -the server). Since UxPlay-1.62, this can be corrected with the -audio offset option -ao x with an -x of about 5.0 (allowed values are decimal numbers between 0 -and 10.0 seconds); this workaround just delays playing of audio on the -server by x seconds, so the effect of pausing or changing -tracks on the client will also be delayed.

    • +

  • In Audio-Only mode the server needs to specify a latency to the +client so the client can show video in sync with audio played on the +server. The default is 0.25 seconds: this can be changed with the +-ao x.y option, where x.y is a decimal like 0.25 in the +range [0.0, 10.0], with microsecond resolution.

    • One common problem involves GStreamer attempting to use incorrectly-configured or absent accelerated hardware h264 video @@ -786,17 +780,11 @@ parameters to be included with the audiosink name. (Some choices of audiosink might not work on your system.)

    -as 0 (or just -a) suppresses playing of streamed audio, but displays streamed video.

    -

    -ao x.y adds an audio offset time in (decimal) -seconds to Audio-only (ALAC) streams to allow synchronization of sound -playing on the UxPlay server with video on the client which delays -playing the audio by x.y seconds (a decimal number). In the -AirPlay Legacy mode used by UxPlay, the client cannot obtain audio -latency information from the server, and appears to assume a latency of -about 5 seconds. This can be compensated for with offset values such as --ao 5 (but the effect of a pause in play etc., on the -client will also be delayed). The -ao option accepts values in the range -[0,10], which it converts to a whole number of milliseconds (-ao 1.2345 -gives 1234 msec audio delay).

    +

    -ao x.y specifies an audio latency) in (decimal) +seconds in Audio-only (ALAC), that is reported to the client so it can +synchronise its video with audio played on the server. Values in the +range [0.0, 10.0] seconds are allowed, and will be converted to a whole +number of microseconds. Default is 0.25 sec (250000 usec).

    -ca filename provides a file (where filename can include a full path) used for output of “cover art” (from Apple Music, etc.,) in audio-only ALAC mode. This @@ -1140,6 +1128,12 @@ as “SupportsLegacyPairing”) of the “features” plist code (reported to the client by the AirPlay server) to be set. The “features” code and other settings are set in UxPlay/lib/dnssdint.h.

    Changelog

    +

    1.63 2023-02-06 Corrected -ao option. Now allows audio latency +reported to client to be changed from default of 0.25 sec (may not be +necessary). Synchronisation of audio on server with video on client in +audio-only ALAC mode now works, as sync=true is now used in the ALAC +GStreamer pipeline. Internal change: all times are now given in +nanoseconds.

    1.62 2023-01-18 Added Audio-only mode time offset -ao x to allow user synchronization of ALAC audio playing on the server with video, song lyrics, etc. playing on the client. x = 5.0 appears to be optimal in diff --git a/README.md b/README.md index f895dbe..27b3287 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# UxPlay 1.62: AirPlay-Mirror and AirPlay-Audio server for Linux, macOS, and Unix (now also runs on Windows). +# UxPlay 1.63: AirPlay-Mirror and AirPlay-Audio server for Linux, macOS, and Unix (now also runs on Windows). ### Now developed at the GitHub site [https://github.com/FDH2/UxPlay](https://github.com/FDH2/UxPlay) (where all user issues should be posted). @@ -338,12 +338,9 @@ run "`uxplay -ca &`" in the background, then run a image viewer with an a is "feh": run "``feh -R 1 ``" in the foreground; terminate feh and then Uxplay with "`ctrl-C fg ctrl-C`". -* If you wish to listen in Audio-Only mode on the server while watching the client screen (for video or Apple Music song lyrics, etc.), -the video on the client is delayed by about 5 seconds behind the the audio on the server (_this is a Legacy mode issue: the client -does not receive latency information to sync its video with audio played on the server_). Since UxPlay-1.62, this can be corrected with -the **audio offset** option `-ao x` with an _x_ of about 5.0 -(allowed values are decimal numbers between 0 and 10.0 seconds); this workaround just delays playing of audio on the server by _x_ seconds, so the effect -of pausing or changing tracks on the client will also be delayed. +* In Audio-Only mode the server needs to specify a latency to the client so the client can show video in sync with audio played +on the server. The default is 0.25 seconds: this can be changed with the `-ao x.y` option, where x.y is a decimal +like 0.25 in the range [0.0, 10.0], with microsecond resolution. **One common problem involves GStreamer attempting to use incorrectly-configured or absent accelerated hardware h264 @@ -664,12 +661,9 @@ which will not work if a firewall is running. **-as 0** (or just **-a**) suppresses playing of streamed audio, but displays streamed video. -**-ao x.y** adds an audio offset time in (decimal) seconds to Audio-only (ALAC) streams to allow synchronization of sound - playing on the UxPlay server with video on the client which delays playing the audio by _x.y_ seconds (a - decimal number). In the AirPlay Legacy mode used by UxPlay, the client cannot obtain audio latency information - from the server, and appears to assume a latency of about 5 seconds. This can be compensated for with offset values such - as `-ao 5` (but the effect of a pause in play etc., on the client will also be delayed). The -ao option accepts - values in the range [0,10], which it converts to a whole number of milliseconds (-ao 1.2345 gives 1234 msec audio delay). +**-ao x.y** specifies an audio latency) in (decimal) seconds in Audio-only (ALAC), that is reported to the client so it + can synchronise its video with audio played on the server. Values in the range [0.0, 10.0] seconds are allowed, and + will be converted to a whole number of microseconds. Default is 0.25 sec (250000 usec). **-ca _filename_** provides a file (where _filename_ can include a full path) used for output of "cover art" (from Apple Music, _etc._,) in audio-only ALAC mode. This file is overwritten with the latest cover art as @@ -945,6 +939,11 @@ tvOS 12.2.1); it seems that the use of "legacy" protocol just requires bit 27 (l The "features" code and other settings are set in `UxPlay/lib/dnssdint.h`. # Changelog +1.63 2023-02-06 Corrected -ao option. Now allows audio latency reported to client to be changed + from default of 0.25 sec (may not be necessary). Synchronisation of audio on server + with video on client in audio-only ALAC mode now works, as sync=true is now used in + the ALAC GStreamer pipeline. Internal change: all times are now given in nanoseconds. + 1.62 2023-01-18 Added Audio-only mode time offset -ao x to allow user synchronization of ALAC audio playing on the server with video, song lyrics, etc. playing on the client. x = 5.0 appears to be optimal in many cases. Quality fixes: cleanup in volume diff --git a/README.txt b/README.txt index b158d6f..1f26d2c 100644 --- a/README.txt +++ b/README.txt @@ -1,4 +1,4 @@ -# UxPlay 1.62: AirPlay-Mirror and AirPlay-Audio server for Linux, macOS, and Unix (now also runs on Windows). +# UxPlay 1.63: AirPlay-Mirror and AirPlay-Audio server for Linux, macOS, and Unix (now also runs on Windows). ### Now developed at the GitHub site (where all user issues should be posted). @@ -404,17 +404,11 @@ for help with this or other problems. "`feh -R 1 `" in the foreground; terminate feh and then Uxplay with "`ctrl-C fg ctrl-C`". -- If you wish to listen in Audio-Only mode on the server while - watching the client screen (for video or Apple Music song lyrics, - etc.), the video on the client is delayed by about 5 seconds behind - the the audio on the server (*this is a Legacy mode issue: the - client does not receive latency information to sync its video with - audio played on the server*). Since UxPlay-1.62, this can be - corrected with the **audio offset** option `-ao x` with an *x* of - about 5.0 (allowed values are decimal numbers between 0 and 10.0 - seconds); this workaround just delays playing of audio on the server - by *x* seconds, so the effect of pausing or changing tracks on the - client will also be delayed. +- In Audio-Only mode the server needs to specify a latency to the + client so the client can show video in sync with audio played on the + server. The default is 0.25 seconds: this can be changed with the + `-ao x.y` option, where x.y is a decimal like 0.25 in the range + \[0.0, 10.0\], with microsecond resolution. **One common problem involves GStreamer attempting to use incorrectly-configured or absent accelerated hardware h264 video @@ -808,16 +802,11 @@ name. (Some choices of audiosink might not work on your system.) **-as 0** (or just **-a**) suppresses playing of streamed audio, but displays streamed video. -**-ao x.y** adds an audio offset time in (decimal) seconds to Audio-only -(ALAC) streams to allow synchronization of sound playing on the UxPlay -server with video on the client which delays playing the audio by *x.y* -seconds (a decimal number). In the AirPlay Legacy mode used by UxPlay, -the client cannot obtain audio latency information from the server, and -appears to assume a latency of about 5 seconds. This can be compensated -for with offset values such as `-ao 5` (but the effect of a pause in -play etc., on the client will also be delayed). The -ao option accepts -values in the range \[0,10\], which it converts to a whole number of -milliseconds (-ao 1.2345 gives 1234 msec audio delay). +**-ao x.y** specifies an audio latency) in (decimal) seconds in +Audio-only (ALAC), that is reported to the client so it can synchronise +its video with audio played on the server. Values in the range \[0.0, +10.0\] seconds are allowed, and will be converted to a whole number of +microseconds. Default is 0.25 sec (250000 usec). **-ca *filename*** provides a file (where *filename* can include a full path) used for output of "cover art" (from Apple Music, *etc.*,) in @@ -1183,6 +1172,12 @@ other settings are set in `UxPlay/lib/dnssdint.h`. # Changelog +1.63 2023-02-06 Corrected -ao option. Now allows audio latency reported +to client to be changed from default of 0.25 sec (may not be necessary). +Synchronisation of audio on server with video on client in audio-only +ALAC mode now works, as sync=true is now used in the ALAC GStreamer +pipeline. Internal change: all times are now given in nanoseconds. + 1.62 2023-01-18 Added Audio-only mode time offset -ao x to allow user synchronization of ALAC audio playing on the server with video, song lyrics, etc. playing on the client. x = 5.0 appears to be optimal in diff --git a/lib/raop.c b/lib/raop.c index d7a8a26..3137b63 100644 --- a/lib/raop.c +++ b/lib/raop.c @@ -60,6 +60,7 @@ struct raop_s { uint8_t overscanned; uint8_t clientFPSdata; + int audio_delay_micros; int max_ntp_timeouts; }; @@ -461,6 +462,7 @@ raop_init(int max_clients, raop_callbacks_t *callbacks) { raop->clientFPSdata = 0; raop->max_ntp_timeouts = 0; + raop->audio_delay_micros = 250000; return raop; } @@ -519,6 +521,11 @@ int raop_set_plist(raop_t *raop, const char *plist_item, const int value) { } else if (strcmp(plist_item, "max_ntp_timeouts") == 0) { raop->max_ntp_timeouts = (value > 0 ? value : 0); if (raop->max_ntp_timeouts != value) retval = 1; + } else if (strcmp(plist_item, "audio_delay_micros") == 0) { + if (value >= 0 && value <= 10 * SECOND_IN_USECS) { + raop->audio_delay_micros = value; + } + if (raop->audio_delay_micros != value) retval = 1; } else { retval = -1; } diff --git a/lib/raop_handlers.h b/lib/raop_handlers.h index 759c72f..a47bb21 100644 --- a/lib/raop_handlers.h +++ b/lib/raop_handlers.h @@ -20,6 +20,8 @@ #include #include #include +#define AUDIO_SAMPLE_RATE 44100 /* all supported AirPlay audio format use this sample rate */ +#define SECOND_IN_USECS 1000000 typedef void (*raop_handler_t)(raop_conn_t *, http_request_t *, http_response_t *, char **, int *); @@ -500,7 +502,9 @@ raop_handler_setup(raop_conn_t *conn, unsigned short cport = conn->raop->control_lport, dport = conn->raop->data_lport; unsigned short remote_cport = 0; unsigned char ct; - unsigned int sr = 44100; /* all AirPlay audio formats supported so far have sample rate 44.1kHz */ + unsigned int sr = AUDIO_SAMPLE_RATE; /* all AirPlay audio formats supported so far have sample rate 44.1kHz */ + unsigned int ad = (unsigned int) (((uint64_t) conn->raop->audio_delay_micros) * AUDIO_SAMPLE_RATE / SECOND_IN_USECS); + uint64_t uint_val = 0; plist_t req_stream_control_port_node = plist_dict_get_item(req_stream_node, "controlPort"); plist_get_uint_val(req_stream_control_port_node, &uint_val); @@ -545,7 +549,7 @@ raop_handler_setup(raop_conn_t *conn, } if (conn->raop_rtp) { - raop_rtp_start_audio(conn->raop_rtp, use_udp, &remote_cport, &cport, &dport, &ct, &sr); + raop_rtp_start_audio(conn->raop_rtp, use_udp, &remote_cport, &cport, &dport, &ct, &sr, &ad); logger_log(conn->raop->logger, LOGGER_DEBUG, "RAOP initialized success"); } else { logger_log(conn->raop->logger, LOGGER_ERR, "RAOP not initialized at SETUP, playing will fail!"); @@ -686,7 +690,10 @@ raop_handler_record(raop_conn_t *conn, http_request_t *request, http_response_t *response, char **response_data, int *response_datalen) { + char audio_latency[12]; + unsigned int ad = (unsigned int) (((uint64_t) conn->raop->audio_delay_micros) * AUDIO_SAMPLE_RATE / SECOND_IN_USECS); + sprintf(audio_latency, "%u", ad); logger_log(conn->raop->logger, LOGGER_DEBUG, "raop_handler_record"); - http_response_add_header(response, "Audio-Latency", "11025"); + http_response_add_header(response, "Audio-Latency", audio_latency); http_response_add_header(response, "Audio-Jack-Status", "connected; type=analog"); } diff --git a/lib/raop_rtp.c b/lib/raop_rtp.c index efed2bc..82570a0 100644 --- a/lib/raop_rtp.c +++ b/lib/raop_rtp.c @@ -54,6 +54,7 @@ struct raop_rtp_s { // Time and sync raop_ntp_t *ntp; double rtp_clock_rate; + unsigned int audio_delay_rtp; int64_t rtp_sync_offset; raop_rtp_sync_data_t sync_data[RAOP_RTP_SYNC_DATA_COUNT]; int sync_data_index; @@ -551,14 +552,12 @@ raop_rtp_thread_udp(void *arg) uint64_t sync_ntp_raw = byteutils_get_long_be(packet, 8); uint64_t sync_ntp_remote = raop_ntp_timestamp_to_nano_seconds(sync_ntp_raw, true); uint64_t sync_ntp_local = raop_ntp_convert_remote_time(raop_rtp->ntp, sync_ntp_remote); - int64_t shift; + int64_t shift = 0; switch (raop_rtp->ct) { - case 0x08: /*AAC-ELD */ - shift = -11025; /* 44100/4 */ + case 0x02: /* ALAC audio-only mode */ break; - case 0x02: default: - shift = 0; /* not needed for ALAC (audio only) */ + shift -= (int64_t) raop_rtp->audio_delay_rtp; /* remove delay in Mirror mode */ break; } char *str = utils_data_to_string(packet, packetlen, 20); @@ -710,8 +709,8 @@ raop_rtp_thread_udp(void *arg) // Start rtp service, three udp ports void -raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, - unsigned short *control_lport, unsigned short *data_lport, unsigned char *ct, unsigned int *sr) +raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, unsigned short *control_lport, + unsigned short *data_lport, unsigned char *ct, unsigned int *sr, unsigned int *ad) { logger_log(raop_rtp->logger, LOGGER_INFO, "raop_rtp starting audio"); int use_ipv6 = 0; @@ -726,6 +725,7 @@ raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_ raop_rtp->ct = *ct; raop_rtp->rtp_clock_rate = SECOND_IN_NSECS / *sr; + raop_rtp->audio_delay_rtp = *ad; /* Initialize ports and sockets */ raop_rtp->control_lport = *control_lport; diff --git a/lib/raop_rtp.h b/lib/raop_rtp.h index a1ca7a3..87751b3 100644 --- a/lib/raop_rtp.h +++ b/lib/raop_rtp.h @@ -29,8 +29,8 @@ typedef struct raop_rtp_s raop_rtp_t; raop_rtp_t *raop_rtp_init(logger_t *logger, raop_callbacks_t *callbacks, raop_ntp_t *ntp, const unsigned char *remote, int remotelen, const unsigned char *aeskey, const unsigned char *aesiv); -void raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, - unsigned short *control_lport, unsigned short *data_lport, unsigned char *ct, unsigned int *sr); +void raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, unsigned short *control_lport, + unsigned short *data_lport, unsigned char *ct, unsigned int *sr, unsigned int *ad); void raop_rtp_set_volume(raop_rtp_t *raop_rtp, float volume); void raop_rtp_set_metadata(raop_rtp_t *raop_rtp, const char *data, int datalen); diff --git a/renderers/audio_renderer.h b/renderers/audio_renderer.h index 00e39c4..adeea75 100644 --- a/renderers/audio_renderer.h +++ b/renderers/audio_renderer.h @@ -30,7 +30,7 @@ extern "C" { #include "../lib/logger.h" bool gstreamer_init(); -void audio_renderer_init(logger_t *logger, const char* audiosink, const char* audiodelay); +void audio_renderer_init(logger_t *logger, const char* audiosink); void audio_renderer_start(unsigned char* compression_type); void audio_renderer_stop(); void audio_renderer_render_buffer(unsigned char* data, int *data_len, unsigned short *seqnum, uint64_t *ntp_time); diff --git a/renderers/audio_renderer_gstreamer.c b/renderers/audio_renderer_gstreamer.c index 9df50bb..239bc16 100644 --- a/renderers/audio_renderer_gstreamer.c +++ b/renderers/audio_renderer_gstreamer.c @@ -82,7 +82,7 @@ static GstClockTime gst_audio_pipeline_base_time = GST_CLOCK_TIME_NONE; static logger_t *logger = NULL; const char * format[NFORMATS]; -void audio_renderer_init(logger_t *render_logger, const char* audiosink, const char* audio_delay) { +void audio_renderer_init(logger_t *render_logger, const char* audiosink) { GError *error = NULL; GstCaps *caps = NULL; GstClock *clock = gst_system_clock_obtain(); @@ -101,11 +101,6 @@ void audio_renderer_init(logger_t *render_logger, const char* audiosink, const c g_string_append(launch, "! avdec_aac ! "); break; case 1: /* ALAC */ - if (audio_delay[0]) { - g_string_append(launch, "min-threshold-time="); - g_string_append(launch, audio_delay); - g_string_append(launch, "000000 "); - } g_string_append(launch, "! avdec_alac ! "); break; case 3: /*PCM*/ @@ -117,7 +112,14 @@ void audio_renderer_init(logger_t *render_logger, const char* audiosink, const c g_string_append (launch, "audioresample ! "); /* wasapisink must resample from 44.1 kHz to 48 kHz */ g_string_append (launch, "volume name=volume ! level ! "); g_string_append (launch, audiosink); - g_string_append (launch, " sync=false"); + switch(i) { + case 1: /*ALAC*/ + g_string_append (launch, " sync=true"); + break; + default: + g_string_append (launch, " sync=false"); + break; + } renderer_type[i]->pipeline = gst_parse_launch(launch->str, &error); if (error) { g_error ("gst_parse_launch error (audio %d):\n %s\n", i+1, error->message); diff --git a/uxplay.1 b/uxplay.1 index 31be5dc..976c085 100644 --- a/uxplay.1 +++ b/uxplay.1 @@ -1,11 +1,11 @@ -.TH UXPLAY "1" "January 2023" "1.62" "User Commands" +.TH UXPLAY "1" "February 2023" "1.63" "User Commands" .SH NAME uxplay \- start AirPlay server .SH SYNOPSIS .B uxplay [\fI\,-n name\/\fR] [\fI\,-s wxh\/\fR] [\fI\,-p \/\fR[\fI\,n\/\fR]] [more \fI OPTIONS \/\fR ...] .SH DESCRIPTION -UxPlay 1.62: An open\-source AirPlay mirroring (+ audio streaming) server. +UxPlay 1.63: An open\-source AirPlay mirroring (+ audio streaming) server. .SH OPTIONS .TP .B @@ -73,7 +73,7 @@ UxPlay 1.62: An open\-source AirPlay mirroring (+ audio streaming) server. .TP \fB\-as\fR 0 (or \fB\-a\fR) Turn audio off, streamed video only. .TP -\fB\-ao\fR x.y Audio offset time in seconds (default 0.0) in Audio-only mode. +\fB\-ao\fR x.y Audio-only mode latency in seconds (default 0.25) used by client. .TP \fB\-ca\fI fn \fR In Airplay Audio (ALAC) mode, write cover-art to file fn. .TP diff --git a/uxplay.cpp b/uxplay.cpp index 747257b..9bec039 100644 --- a/uxplay.cpp +++ b/uxplay.cpp @@ -51,8 +51,9 @@ #include "renderers/video_renderer.h" #include "renderers/audio_renderer.h" -#define VERSION "1.62" +#define VERSION "1.63" +#define SECOND_IN_USECS 1000000 #define DEFAULT_NAME "UxPlay" #define DEFAULT_DEBUG_LOG false #define LOWEST_ALLOWED_PORT 1024 @@ -73,7 +74,7 @@ static videoflip_t videoflip[2] = { NONE , NONE }; static bool use_video = true; static unsigned char compression_type = 0; static std::string audiosink = "autoaudiosink"; -static std::string audiodelay = ""; +static int audiodelay = -1; static bool use_audio = true; static bool new_window_closing_behavior = true; static bool close_window; @@ -387,7 +388,7 @@ static void print_info (char *name) { printf(" some choices:pulsesink,alsasink,pipewiresink,jackaudiosink,\n"); printf(" osssink,oss4sink,osxaudiosink,wasapisink,directsoundsink.\n"); printf("-as 0 (or -a) Turn audio off, streamed video only\n"); - printf("-ao x.y Audio offset time in seconds (default 0.0) in Audio-only mode.\n"); + printf("-ao x.y Audio-only mode latency in seconds (default 0.25) used by client.\n"); printf("-ca In Airplay Audio (ALAC) mode, write cover-art to file \n"); printf("-reset n Reset after 3n seconds client silence (default %d, 0=never)\n", NTP_TIMEOUT_LIMIT); printf("-nc do Not Close video window when client stops mirroring\n"); @@ -741,20 +742,14 @@ static void parse_arguments (int argc, char *argv[]) { int n; char *end; if (i < argc - 1 && *argv[i+1] != '-') { - n = (int) (1000 * strtof(argv[++i], &end)); - if (*end == '\0' && n >=0 && n <= 10000) { - audiodelay.erase(); - if (n > 0) { - char* delay = new char[6]; - snprintf(delay, 6, "%d", n); - audiodelay = delay; - delete[] delay; - } + n = (int) (strtof(argv[++i], &end) * SECOND_IN_USECS); + if (*end == '\0' && n >=0 && n <= 10 * SECOND_IN_USECS) { + audiodelay = n; continue; } } fprintf(stderr, "invalid argument -ao %s: must be a decimal time offset in seconds, range [0,10]\n" - "(like 5 or 4.8, which will be converted to a whole number of milliseconds)\n", argv[i]); + "(like 5 or 4.8, which will be converted to a whole number of microseconds)\n", argv[i]); exit(1); } else { fprintf(stderr, "unknown option %s, stopping\n",argv[i]); @@ -1162,6 +1157,7 @@ int start_raop_server (unsigned short display[5], unsigned short tcp[3], unsigne if (show_client_FPS_data) raop_set_plist(raop, "clientFPSdata", 1); raop_set_plist(raop, "max_ntp_timeouts", max_ntp_timeouts); + if (audiodelay >= 0) raop_set_plist(raop, "audio_delay_micros", audiodelay); /* network port selection (ports listed as "0" will be dynamically assigned) */ raop_set_tcp_ports(raop, tcp); @@ -1284,10 +1280,10 @@ int main (int argc, char *argv[]) { logger_set_level(render_logger, debug_log ? LOGGER_DEBUG : LOGGER_INFO); if (use_audio) { - if (audiodelay.c_str()[0]) { - LOGI("Audio-only ALAC streams will be delayed by %s milliseconds", audiodelay.c_str()); + if (audiodelay >= 0) { + LOGI("Audio-only ALAC streams will be delayed by %d microseconds", audiodelay); } - audio_renderer_init(render_logger, audiosink.c_str(), audiodelay.c_str()); + audio_renderer_init(render_logger, audiosink.c_str()); } else { LOGI("audio_disabled"); } From dfd98efbab3d61d0d54f9d74636b0923cfb41baf Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Tue, 7 Feb 2023 14:33:17 -0500 Subject: [PATCH 4/7] cleanups of initial latency code. ntp and rtp times --- lib/raop_buffer.c | 13 ++-- lib/raop_buffer.h | 4 +- lib/raop_handlers.h | 2 +- lib/raop_rtp.c | 156 ++++++++++++++++++++++---------------------- 4 files changed, 90 insertions(+), 85 deletions(-) diff --git a/lib/raop_buffer.c b/lib/raop_buffer.c index fce2d60..b0728ef 100644 --- a/lib/raop_buffer.c +++ b/lib/raop_buffer.c @@ -38,7 +38,8 @@ typedef struct { /* RTP header */ unsigned short seqnum; - uint64_t timestamp; + uint64_t rtp_timestamp; + uint64_t ntp_timestamp; /* Payload data */ unsigned int payload_size; @@ -206,7 +207,7 @@ raop_buffer_decrypt(raop_buffer_t *raop_buffer, unsigned char *data, unsigned ch } int -raop_buffer_enqueue(raop_buffer_t *raop_buffer, unsigned char *data, unsigned short datalen, uint64_t timestamp, int use_seqnum) { +raop_buffer_enqueue(raop_buffer_t *raop_buffer, unsigned char *data, unsigned short datalen, uint64_t *ntp_timestamp, uint64_t *rtp_timestamp, int use_seqnum) { unsigned char empty_packet_marker[] = { 0x00, 0x68, 0x34, 0x00 }; assert(raop_buffer); @@ -247,7 +248,8 @@ raop_buffer_enqueue(raop_buffer_t *raop_buffer, unsigned char *data, unsigned sh /* Update the raop_buffer entry header */ entry->seqnum = seqnum; - entry->timestamp = timestamp; + entry->rtp_timestamp = *rtp_timestamp; + entry->ntp_timestamp = *ntp_timestamp; entry->filled = 1; entry->payload_data = malloc(payload_size); @@ -268,7 +270,7 @@ raop_buffer_enqueue(raop_buffer_t *raop_buffer, unsigned char *data, unsigned sh } void * -raop_buffer_dequeue(raop_buffer_t *raop_buffer, unsigned int *length, uint64_t *timestamp, unsigned short *seqnum, int no_resend) { +raop_buffer_dequeue(raop_buffer_t *raop_buffer, unsigned int *length, uint64_t *ntp_timestamp, uint64_t *rtp_timestamp, unsigned short *seqnum, int no_resend) { assert(raop_buffer); /* Calculate number of entries in the current buffer */ @@ -300,7 +302,8 @@ raop_buffer_dequeue(raop_buffer_t *raop_buffer, unsigned int *length, uint64_t * entry->filled = 0; /* Return entry payload buffer */ - *timestamp = entry->timestamp; + *rtp_timestamp = entry->rtp_timestamp; + *ntp_timestamp = entry->ntp_timestamp; *seqnum = entry->seqnum; *length = entry->payload_size; entry->payload_size = 0; diff --git a/lib/raop_buffer.h b/lib/raop_buffer.h index 0418509..3905793 100644 --- a/lib/raop_buffer.h +++ b/lib/raop_buffer.h @@ -25,8 +25,8 @@ typedef int (*raop_resend_cb_t)(void *opaque, unsigned short seqno, unsigned sho raop_buffer_t *raop_buffer_init(logger_t *logger, const unsigned char *aeskey, const unsigned char *aesiv); -int raop_buffer_enqueue(raop_buffer_t *raop_buffer, unsigned char *data, unsigned short datalen, uint64_t timestamp, int use_seqnum); -void *raop_buffer_dequeue(raop_buffer_t *raop_buffer, unsigned int *length, uint64_t *timestamp, unsigned short *seqnum, int no_resend); +int raop_buffer_enqueue(raop_buffer_t *raop_buffer, unsigned char *data, unsigned short datalen, uint64_t *ntp_timestamp, uint64_t *rtp_timestamp, int use_seqnum); +void *raop_buffer_dequeue(raop_buffer_t *raop_buffer, unsigned int *length, uint64_t *ntp_timestamp, uint64_t *rtp_timestamp, unsigned short *seqnum, int no_resend); void raop_buffer_handle_resends(raop_buffer_t *raop_buffer, raop_resend_cb_t resend_cb, void *opaque); void raop_buffer_flush(raop_buffer_t *raop_buffer, int next_seq); diff --git a/lib/raop_handlers.h b/lib/raop_handlers.h index a47bb21..b7c80be 100644 --- a/lib/raop_handlers.h +++ b/lib/raop_handlers.h @@ -503,7 +503,7 @@ raop_handler_setup(raop_conn_t *conn, unsigned short remote_cport = 0; unsigned char ct; unsigned int sr = AUDIO_SAMPLE_RATE; /* all AirPlay audio formats supported so far have sample rate 44.1kHz */ - unsigned int ad = (unsigned int) (((uint64_t) conn->raop->audio_delay_micros) * AUDIO_SAMPLE_RATE / SECOND_IN_USECS); + unsigned int ad = (unsigned int) conn->raop->audio_delay_micros; uint64_t uint_val = 0; plist_t req_stream_control_port_node = plist_dict_get_item(req_stream_node, "controlPort"); diff --git a/lib/raop_rtp.c b/lib/raop_rtp.c index 82570a0..3c37710 100644 --- a/lib/raop_rtp.c +++ b/lib/raop_rtp.c @@ -36,8 +36,8 @@ #define RAOP_RTP_SYNC_DATA_COUNT 8 #define SEC SECOND_IN_NSECS -#define DELAY_AAC 500000000 //empirical, matches audio latency of about -0.5 sec after first clock sync event -#define DELAY_ALAC 200000000 //empirical, matches audio latency of about -2.0 sec after first clock sync event +#define DELAY_AAC 0.25 //empirical, matches audio latency of about -0.25 sec after first clock sync event +#define DELAY_ALAC 2.0 //empirical, matches audio latency of about -2.0 sec after first clock sync event /* note: it is unclear what will happen in the unlikely event that this code is running at the time of the unix-time * epoch event on 2038-01-19 at 3:14:08 UTC ! (but Apple will surely have removed AirPlay "legacy pairing" by then!) */ @@ -54,7 +54,7 @@ struct raop_rtp_s { // Time and sync raop_ntp_t *ntp; double rtp_clock_rate; - unsigned int audio_delay_rtp; + unsigned int audio_delay_micros; int64_t rtp_sync_offset; raop_rtp_sync_data_t sync_data[RAOP_RTP_SYNC_DATA_COUNT]; int sync_data_index; @@ -388,34 +388,35 @@ raop_rtp_process_events(raop_rtp_t *raop_rtp, void *cb_data) return 0; } -void raop_rtp_sync_clock(raop_rtp_t *raop_rtp, uint64_t ntp_time, uint64_t rtp_time, int shift) { - int latest; - uint32_t valid_data_count = 0; - valid_data_count = 0; - double total_offsets = 0; - int64_t rtp_offset, avg_offset, correction; +void raop_rtp_sync_clock(raop_rtp_t *raop_rtp, uint64_t *ntp_time, uint64_t *rtp_time) { + /* ntp_time = (uint64_t)(((int64_t)(raop_rtp->rtp_clock_rate * rtp_time)) + raop_rtp->rtp_sync_offset) */ + int latest, valid_data_count = 0; + uint64_t ntp_sum = 0, rtp_sum = 0; + double offset = ((double) *ntp_time) - raop_rtp->rtp_clock_rate * *rtp_time; + int64_t correction = 0; raop_rtp->sync_data_index = (raop_rtp->sync_data_index + 1) % RAOP_RTP_SYNC_DATA_COUNT; latest = raop_rtp->sync_data_index; - raop_rtp->sync_data[latest].rtp_time = rtp_time; - raop_rtp->sync_data[latest].ntp_time = ntp_time; + raop_rtp->sync_data[latest].rtp_time = *rtp_time; + raop_rtp->sync_data[latest].ntp_time = *ntp_time; for (int i = 0; i < RAOP_RTP_SYNC_DATA_COUNT; i++) { if (raop_rtp->sync_data[i].ntp_time == 0) continue; - rtp_offset = ((int64_t) raop_rtp->sync_data[i].rtp_time) - ((int64_t) raop_rtp->sync_data[latest].rtp_time); - total_offsets += ((double) rtp_offset) * raop_rtp-> rtp_clock_rate; - total_offsets -= (double) (((int64_t) raop_rtp->sync_data[i].ntp_time) - ((int64_t) raop_rtp->sync_data[latest].ntp_time)); valid_data_count++; + if (i == latest) continue; + ntp_sum += *ntp_time - raop_rtp->sync_data[i].ntp_time; + rtp_sum += *rtp_time - raop_rtp->sync_data[i].rtp_time; } - total_offsets = (total_offsets / valid_data_count); - rtp_offset = ((int64_t) raop_rtp->sync_data[latest].rtp_time) - ((int64_t) raop_rtp->rtp_start_time) + ((int64_t) shift); - total_offsets += ((double) rtp_offset) * raop_rtp->rtp_clock_rate; - avg_offset = (int64_t) total_offsets; - avg_offset -= ((int64_t) raop_rtp->sync_data[latest].ntp_time) - ((int64_t) raop_rtp->ntp_start_time); - correction = avg_offset - raop_rtp->rtp_sync_offset; - raop_rtp->rtp_sync_offset = avg_offset; - logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp sync correction=%lld, rtp_sync_offset = %8.6f ", - correction, ((double) raop_rtp->rtp_sync_offset) / SEC); + + if (valid_data_count > 1) { + correction -= raop_rtp->rtp_sync_offset; + offset += (((double) ntp_sum) - raop_rtp->rtp_clock_rate * rtp_sum) / valid_data_count; + } + raop_rtp->rtp_sync_offset = (int64_t) offset; + correction += raop_rtp->rtp_sync_offset; + + logger_log(raop_rtp->logger, LOGGER_DEBUG, "dataset %d raop_rtp sync correction=%lld, rtp_sync_offset = %8.6f ", + valid_data_count, correction, offset); } uint64_t rtp64_time (raop_rtp_t *raop_rtp, const uint32_t *rtp32) { @@ -451,14 +452,15 @@ raop_rtp_thread_udp(void *arg) struct sockaddr_storage saddr; socklen_t saddrlen; + double latency = 0; + /* for initial rtp to ntp conversions */ bool have_synced = false; int rtp_count = 0; - int64_t initial_offset = 0; double sync_adjustment = 0; - int64_t delay = 0; + uint64_t delay = 0; unsigned short seqnum1 = 0, seqnum2 = 0; - bool offset_estimate_initialized = false; + assert(raop_rtp); raop_rtp->ntp_start_time = raop_ntp_get_local_time(raop_rtp->ntp); @@ -469,7 +471,22 @@ raop_rtp_thread_udp(void *arg) logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp start_time = %8.6f (raop_rtp audio)", ((double) raop_rtp->ntp_start_time) / SEC); - + + switch (raop_rtp->ct) { + case 0x02: + latency = DELAY_ALAC; + delay = (uint64_t)(latency * SECOND_IN_NSECS); /* DELAY = 2.0 sec is empirical choice for ALAC */ + logger_log(raop_rtp->logger, LOGGER_DEBUG, "Audio is ALAC: using initial latency estimate -%8.6f sec", latency); + break; + case 0x08: + latency = DELAY_AAC; + delay = (uint64_t)(latency * SECOND_IN_NSECS); /* DELAY = 0.25 sec is empirical choice for AAC-ELD */ + logger_log(raop_rtp->logger, LOGGER_DEBUG, "Audio is AAC: using initial latency estimate -%8.6f sec", latency); + break; + default: + break; + } + while(1) { fd_set rfds; struct timeval tv; @@ -520,8 +537,13 @@ raop_rtp_thread_udp(void *arg) if (resent_packetlen >= 12) { uint32_t timestamp = byteutils_get_int_be(resent_packet, 4); uint64_t rtp_time = rtp64_time(raop_rtp, ×tamp); + uint64_t ntp_time = 0; + if (have_synced) { + ntp_time = (uint64_t) (raop_rtp->rtp_sync_offset + (int64_t) (raop_rtp->rtp_clock_rate * rtp_time)); + ntp_time = raop_ntp_convert_remote_time(raop_rtp->ntp, ntp_time); + } logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp resent audio packet: seqnum=%u", seqnum); - int result = raop_buffer_enqueue(raop_rtp->buffer, resent_packet, resent_packetlen, rtp_time, 1); + int result = raop_buffer_enqueue(raop_rtp->buffer, resent_packet, resent_packetlen, &ntp_time, &rtp_time, 1); assert(result >= 0); } else { /* type_c = 0x56 packets with length 8 have been reported */ @@ -543,8 +565,6 @@ raop_rtp_thread_udp(void *arg) // The unit for the rtp clock is 1 / sample rate = 1 / 44100 uint32_t sync_rtp = byteutils_get_int_be(packet, 4); uint64_t sync_rtp64 = rtp64_time(raop_rtp, &sync_rtp); - - if (have_synced == false) { logger_log(raop_rtp->logger, LOGGER_DEBUG, "first audio rtp sync"); have_synced = true; @@ -552,21 +572,13 @@ raop_rtp_thread_udp(void *arg) uint64_t sync_ntp_raw = byteutils_get_long_be(packet, 8); uint64_t sync_ntp_remote = raop_ntp_timestamp_to_nano_seconds(sync_ntp_raw, true); uint64_t sync_ntp_local = raop_ntp_convert_remote_time(raop_rtp->ntp, sync_ntp_remote); - int64_t shift = 0; - switch (raop_rtp->ct) { - case 0x02: /* ALAC audio-only mode */ - break; - default: - shift -= (int64_t) raop_rtp->audio_delay_rtp; /* remove delay in Mirror mode */ - break; - } char *str = utils_data_to_string(packet, packetlen, 20); logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp sync: client ntp=%8.6f, ntp = %8.6f, ntp_start_time %8.6f, sync_rtp=%u\n%s", ((double) sync_ntp_remote) / SEC, ((double)sync_ntp_local) / SEC, ((double) raop_rtp->ntp_start_time) / SEC, sync_rtp, str); free(str); - raop_rtp_sync_clock(raop_rtp, sync_ntp_local, sync_rtp64, shift); + raop_rtp_sync_clock(raop_rtp, &sync_ntp_remote, &sync_rtp64); } else { char *str = utils_data_to_string(packet, packetlen, 16); logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp unknown udp control packet\n%s", str); @@ -619,69 +631,59 @@ raop_rtp_thread_udp(void *arg) int no_resend = (raop_rtp->control_rport == 0); /* true when control_rport is not set */ uint32_t rtp_timestamp = byteutils_get_int_be(packet, 4); uint64_t rtp_time = rtp64_time(raop_rtp, &rtp_timestamp); + uint64_t ntp_time = 0; if (have_synced == false) { /* until the first rtp sync occurs, we don't know the exact client ntp timestamp that matches the client rtp timestamp */ int64_t sync_ntp = ((int64_t) raop_ntp_get_local_time(raop_rtp->ntp)) - ((int64_t) raop_rtp->ntp_start_time) ; int64_t sync_rtp = ((int64_t) rtp_time) - ((int64_t) raop_rtp->rtp_start_time); - int64_t offset; unsigned short seqnum = byteutils_get_short_be(packet,2); - if (!offset_estimate_initialized) { - offset_estimate_initialized = true; - switch (raop_rtp->ct) { - case 0x02: - delay = DELAY_ALAC; /* DELAY = 2000000000 (2.0 sec) is empirical choice for ALAC */ - logger_log(raop_rtp->logger, LOGGER_DEBUG, "Audio is ALAC: using initial latency estimate -%8.6f sec", - ((double) delay) / SEC); - break; - case 0x08: - delay = DELAY_AAC; /* DELAY = 500000000 (0.5 sec) is empirical choice for AAC-ELD */ - logger_log(raop_rtp->logger, LOGGER_DEBUG, "Audio is AAC: using initial latency estimate -%8.6f sec", - ((double) delay ) / SEC); - break; - default: - break; - } - initial_offset = -(sync_ntp + delay); - raop_rtp->rtp_sync_offset = initial_offset; - sync_adjustment = 0; + if (rtp_count == 0) { + sync_adjustment = ((double) sync_ntp); + rtp_count = 1; seqnum1 = seqnum; seqnum2 = seqnum; } - sync_ntp += delay; - offset = -sync_ntp; - if (seqnum2 != seqnum) { /* for AAC-ELD only use copy 3 of the 3 copies of each frame */ + if (seqnum2 != seqnum) { /* for AAC-ELD only use copy 1 of the 3 copies of each frame */ rtp_count++; - offset -= initial_offset; - sync_adjustment += ((double) offset) + (((double) sync_rtp) * raop_rtp->rtp_clock_rate); - raop_rtp->rtp_sync_offset = initial_offset + (int64_t) (sync_adjustment / rtp_count); - //logger_log(raop_rtp->logger, LOGGER_DEBUG, "initial estimate of rtp_sync_offset %d secnum = %u: %8.6f", - // rtp_count, seqnum, ((double) raop_rtp->rtp_sync_offset) / SEC); + sync_adjustment += (((double) sync_ntp) - raop_rtp->rtp_clock_rate * sync_rtp - sync_adjustment) / rtp_count; } seqnum2 = seqnum1; seqnum1 = seqnum; + } else { + ntp_time = (uint64_t) (raop_rtp->rtp_sync_offset + (int64_t) (raop_rtp->rtp_clock_rate * rtp_time)); + ntp_time = raop_ntp_convert_remote_time(raop_rtp->ntp, ntp_time); } - int result = raop_buffer_enqueue(raop_rtp->buffer, packet, packetlen, rtp_time, 1); + int result = raop_buffer_enqueue(raop_rtp->buffer, packet, packetlen, &ntp_time, &rtp_time, 1); assert(result >= 0); // Render continuous buffer entries void *payload = NULL; unsigned int payload_size; unsigned short seqnum; uint64_t rtp64_timestamp; - while ((payload = raop_buffer_dequeue(raop_rtp->buffer, &payload_size, &rtp64_timestamp, &seqnum, no_resend))) { - double elapsed_time = (((double) (rtp64_timestamp - (uint64_t) raop_rtp->rtp_start_time)) * raop_rtp->rtp_clock_rate); + uint64_t ntp_timestamp; + while ((payload = raop_buffer_dequeue(raop_rtp->buffer, &payload_size, &ntp_timestamp, &rtp64_timestamp, &seqnum, no_resend))) { audio_decode_struct audio_data; - audio_data.data_len = payload_size; - audio_data.data = payload; - audio_data.ntp_time = raop_rtp->ntp_start_time + (uint64_t) elapsed_time; - audio_data.ntp_time -= raop_rtp->rtp_sync_offset; audio_data.rtp_time = rtp64_timestamp; audio_data.seqnum = seqnum; + audio_data.data_len = payload_size; + audio_data.data = payload; + if (ntp_timestamp) { + audio_data.ntp_time = ntp_timestamp; + } else { + if (have_synced) { + uint64_t ntp_remote = (uint64_t) (raop_rtp->rtp_sync_offset + (int64_t) (raop_rtp->rtp_clock_rate * rtp64_timestamp)); + audio_data.ntp_time = raop_ntp_convert_remote_time(raop_rtp->ntp, ntp_remote); + } else { + audio_data.ntp_time = raop_rtp->ntp_start_time + delay + + (uint64_t) (sync_adjustment + raop_rtp->rtp_clock_rate * (rtp64_timestamp - raop_rtp->rtp_start_time)); + } + } raop_rtp->callbacks.audio_process(raop_rtp->callbacks.cls, raop_rtp->ntp, &audio_data); free(payload); uint64_t ntp_now = raop_ntp_get_local_time(raop_rtp->ntp); - int64_t latency = ((int64_t) ntp_now) - ((int64_t) audio_data.ntp_time); + int64_t latency = ((int64_t) ntp_now) - ((int64_t) audio_data.ntp_time); logger_log(raop_rtp->logger, LOGGER_DEBUG, "raop_rtp audio: now = %8.6f, ntp = %8.6f, latency = %8.6f, rtp_time=%u seqnum = %u", - ((double) ntp_now ) / SEC, ((double) audio_data.ntp_time) / SEC, ((double) latency) / SEC, (uint32_t) rtp64_timestamp, + ((double) ntp_now ) / SEC, ((double) audio_data.ntp_time) / SEC, ((double) latency) / SEC, (uint32_t) rtp64_timestamp, seqnum); } @@ -725,7 +727,7 @@ raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_ raop_rtp->ct = *ct; raop_rtp->rtp_clock_rate = SECOND_IN_NSECS / *sr; - raop_rtp->audio_delay_rtp = *ad; + raop_rtp->audio_delay_micros = *ad; /* Initialize ports and sockets */ raop_rtp->control_lport = *control_lport; From ad451b4c339bc0f5f36019b0572b8c9d02c57f48 Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Tue, 7 Feb 2023 23:54:23 -0500 Subject: [PATCH 5/7] remove unnecessary "audio delay" feature. --- README.html | 17 +++++++---------- README.md | 11 ++++------- README.txt | 18 +++++++----------- lib/raop_handlers.h | 3 +-- lib/raop_rtp.c | 4 +--- lib/raop_rtp.h | 2 +- uxplay.1 | 2 +- uxplay.cpp | 9 +++------ 8 files changed, 25 insertions(+), 41 deletions(-) diff --git a/README.html b/README.html index d588998..ca77351 100644 --- a/README.html +++ b/README.html @@ -400,11 +400,6 @@ from sources like Apple Music in Audio-Only (ALAC) mode: run a image viewer with an autoreload feature: an example is “feh”: run “feh -R 1 <name>” in the foreground; terminate feh and then Uxplay with “ctrl-C fg ctrl-C”.

    -

  • In Audio-Only mode the server needs to specify a latency to the -client so the client can show video in sync with audio played on the -server. The default is 0.25 seconds: this can be changed with the --ao x.y option, where x.y is a decimal like 0.25 in the -range [0.0, 10.0], with microsecond resolution.

    • One common problem involves GStreamer attempting to use incorrectly-configured or absent accelerated hardware h264 video @@ -780,11 +775,13 @@ parameters to be included with the audiosink name. (Some choices of audiosink might not work on your system.)

    -as 0 (or just -a) suppresses playing of streamed audio, but displays streamed video.

    -

    -ao x.y specifies an audio latency) in (decimal) -seconds in Audio-only (ALAC), that is reported to the client so it can -synchronise its video with audio played on the server. Values in the -range [0.0, 10.0] seconds are allowed, and will be converted to a whole -number of microseconds. Default is 0.25 sec (250000 usec).

    +

    -al x specifies an audio latency x +in (decimal) seconds in Audio-only (ALAC), that is reported to the +client. Values in the range [0.0, 10.0] seconds are allowed, and will be +converted to a whole number of microseconds. Default is 0.25 sec (250000 +usec). (This replaces the -ao option introduced in v1.62, +as a workaround for a problem that is now fixed: it is not clear if +changing the value of x produces any effects, however.)

    -ca filename provides a file (where filename can include a full path) used for output of “cover art” (from Apple Music, etc.,) in audio-only ALAC mode. This diff --git a/README.md b/README.md index 27b3287..ff0b3a8 100644 --- a/README.md +++ b/README.md @@ -338,10 +338,6 @@ run "`uxplay -ca &`" in the background, then run a image viewer with an a is "feh": run "``feh -R 1 ``" in the foreground; terminate feh and then Uxplay with "`ctrl-C fg ctrl-C`". -* In Audio-Only mode the server needs to specify a latency to the client so the client can show video in sync with audio played -on the server. The default is 0.25 seconds: this can be changed with the `-ao x.y` option, where x.y is a decimal -like 0.25 in the range [0.0, 10.0], with microsecond resolution. - **One common problem involves GStreamer attempting to use incorrectly-configured or absent accelerated hardware h264 video decoding (e.g., VAAPI). @@ -661,9 +657,10 @@ which will not work if a firewall is running. **-as 0** (or just **-a**) suppresses playing of streamed audio, but displays streamed video. -**-ao x.y** specifies an audio latency) in (decimal) seconds in Audio-only (ALAC), that is reported to the client so it - can synchronise its video with audio played on the server. Values in the range [0.0, 10.0] seconds are allowed, and - will be converted to a whole number of microseconds. Default is 0.25 sec (250000 usec). +**-al _x_** specifies an audio latency _x_ in (decimal) seconds in Audio-only (ALAC), that is reported to the client. Values + in the range [0.0, 10.0] seconds are allowed, and will be converted to a whole number of microseconds. Default + is 0.25 sec (250000 usec). (This replaces the `-ao` option introduced in v1.62, as a workaround for a problem that + is now fixed: it is not clear if changing the value of _x_ produces any effects, however.) **-ca _filename_** provides a file (where _filename_ can include a full path) used for output of "cover art" (from Apple Music, _etc._,) in audio-only ALAC mode. This file is overwritten with the latest cover art as diff --git a/README.txt b/README.txt index 1f26d2c..6bf3857 100644 --- a/README.txt +++ b/README.txt @@ -404,12 +404,6 @@ for help with this or other problems. "`feh -R 1 `" in the foreground; terminate feh and then Uxplay with "`ctrl-C fg ctrl-C`". -- In Audio-Only mode the server needs to specify a latency to the - client so the client can show video in sync with audio played on the - server. The default is 0.25 seconds: this can be changed with the - `-ao x.y` option, where x.y is a decimal like 0.25 in the range - \[0.0, 10.0\], with microsecond resolution. - **One common problem involves GStreamer attempting to use incorrectly-configured or absent accelerated hardware h264 video decoding (e.g., VAAPI). Try "`uxplay -avdec`" to force software video @@ -802,11 +796,13 @@ name. (Some choices of audiosink might not work on your system.) **-as 0** (or just **-a**) suppresses playing of streamed audio, but displays streamed video. -**-ao x.y** specifies an audio latency) in (decimal) seconds in -Audio-only (ALAC), that is reported to the client so it can synchronise -its video with audio played on the server. Values in the range \[0.0, -10.0\] seconds are allowed, and will be converted to a whole number of -microseconds. Default is 0.25 sec (250000 usec). +**-al *x*** specifies an audio latency *x* in (decimal) seconds in +Audio-only (ALAC), that is reported to the client. Values in the range +\[0.0, 10.0\] seconds are allowed, and will be converted to a whole +number of microseconds. Default is 0.25 sec (250000 usec). (This +replaces the `-ao` option introduced in v1.62, as a workaround for a +problem that is now fixed: it is not clear if changing the value of *x* +produces any effects, however.) **-ca *filename*** provides a file (where *filename* can include a full path) used for output of "cover art" (from Apple Music, *etc.*,) in diff --git a/lib/raop_handlers.h b/lib/raop_handlers.h index b7c80be..261e3aa 100644 --- a/lib/raop_handlers.h +++ b/lib/raop_handlers.h @@ -503,7 +503,6 @@ raop_handler_setup(raop_conn_t *conn, unsigned short remote_cport = 0; unsigned char ct; unsigned int sr = AUDIO_SAMPLE_RATE; /* all AirPlay audio formats supported so far have sample rate 44.1kHz */ - unsigned int ad = (unsigned int) conn->raop->audio_delay_micros; uint64_t uint_val = 0; plist_t req_stream_control_port_node = plist_dict_get_item(req_stream_node, "controlPort"); @@ -549,7 +548,7 @@ raop_handler_setup(raop_conn_t *conn, } if (conn->raop_rtp) { - raop_rtp_start_audio(conn->raop_rtp, use_udp, &remote_cport, &cport, &dport, &ct, &sr, &ad); + raop_rtp_start_audio(conn->raop_rtp, use_udp, &remote_cport, &cport, &dport, &ct, &sr); logger_log(conn->raop->logger, LOGGER_DEBUG, "RAOP initialized success"); } else { logger_log(conn->raop->logger, LOGGER_ERR, "RAOP not initialized at SETUP, playing will fail!"); diff --git a/lib/raop_rtp.c b/lib/raop_rtp.c index 3c37710..2d7979e 100644 --- a/lib/raop_rtp.c +++ b/lib/raop_rtp.c @@ -54,7 +54,6 @@ struct raop_rtp_s { // Time and sync raop_ntp_t *ntp; double rtp_clock_rate; - unsigned int audio_delay_micros; int64_t rtp_sync_offset; raop_rtp_sync_data_t sync_data[RAOP_RTP_SYNC_DATA_COUNT]; int sync_data_index; @@ -712,7 +711,7 @@ raop_rtp_thread_udp(void *arg) // Start rtp service, three udp ports void raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, unsigned short *control_lport, - unsigned short *data_lport, unsigned char *ct, unsigned int *sr, unsigned int *ad) + unsigned short *data_lport, unsigned char *ct, unsigned int *sr) { logger_log(raop_rtp->logger, LOGGER_INFO, "raop_rtp starting audio"); int use_ipv6 = 0; @@ -727,7 +726,6 @@ raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_ raop_rtp->ct = *ct; raop_rtp->rtp_clock_rate = SECOND_IN_NSECS / *sr; - raop_rtp->audio_delay_micros = *ad; /* Initialize ports and sockets */ raop_rtp->control_lport = *control_lport; diff --git a/lib/raop_rtp.h b/lib/raop_rtp.h index 87751b3..675d619 100644 --- a/lib/raop_rtp.h +++ b/lib/raop_rtp.h @@ -30,7 +30,7 @@ raop_rtp_t *raop_rtp_init(logger_t *logger, raop_callbacks_t *callbacks, raop_nt int remotelen, const unsigned char *aeskey, const unsigned char *aesiv); void raop_rtp_start_audio(raop_rtp_t *raop_rtp, int use_udp, unsigned short *control_rport, unsigned short *control_lport, - unsigned short *data_lport, unsigned char *ct, unsigned int *sr, unsigned int *ad); + unsigned short *data_lport, unsigned char *ct, unsigned int *sr); void raop_rtp_set_volume(raop_rtp_t *raop_rtp, float volume); void raop_rtp_set_metadata(raop_rtp_t *raop_rtp, const char *data, int datalen); diff --git a/uxplay.1 b/uxplay.1 index 976c085..f91280f 100644 --- a/uxplay.1 +++ b/uxplay.1 @@ -73,7 +73,7 @@ UxPlay 1.63: An open\-source AirPlay mirroring (+ audio streaming) server. .TP \fB\-as\fR 0 (or \fB\-a\fR) Turn audio off, streamed video only. .TP -\fB\-ao\fR x.y Audio-only mode latency in seconds (default 0.25) used by client. +\fB\-al\fR x Audio latency in seconds (default 0.25) reported to client. .TP \fB\-ca\fI fn \fR In Airplay Audio (ALAC) mode, write cover-art to file fn. .TP diff --git a/uxplay.cpp b/uxplay.cpp index 9bec039..e97a4fd 100644 --- a/uxplay.cpp +++ b/uxplay.cpp @@ -388,7 +388,7 @@ static void print_info (char *name) { printf(" some choices:pulsesink,alsasink,pipewiresink,jackaudiosink,\n"); printf(" osssink,oss4sink,osxaudiosink,wasapisink,directsoundsink.\n"); printf("-as 0 (or -a) Turn audio off, streamed video only\n"); - printf("-ao x.y Audio-only mode latency in seconds (default 0.25) used by client.\n"); + printf("-al x Audio latency in seconds (default 0.25) reported to client.\n"); printf("-ca In Airplay Audio (ALAC) mode, write cover-art to file \n"); printf("-reset n Reset after 3n seconds client silence (default %d, 0=never)\n", NTP_TIMEOUT_LIMIT); printf("-nc do Not Close video window when client stops mirroring\n"); @@ -738,7 +738,7 @@ static void parse_arguments (int argc, char *argv[]) { bt709_fix = true; } else if (arg == "-nohold") { max_connections = 3; - } else if (arg == "-ao") { + } else if (arg == "-al") { int n; char *end; if (i < argc - 1 && *argv[i+1] != '-') { @@ -748,7 +748,7 @@ static void parse_arguments (int argc, char *argv[]) { continue; } } - fprintf(stderr, "invalid argument -ao %s: must be a decimal time offset in seconds, range [0,10]\n" + fprintf(stderr, "invalid argument -al %s: must be a decimal time offset in seconds, range [0,10]\n" "(like 5 or 4.8, which will be converted to a whole number of microseconds)\n", argv[i]); exit(1); } else { @@ -1280,9 +1280,6 @@ int main (int argc, char *argv[]) { logger_set_level(render_logger, debug_log ? LOGGER_DEBUG : LOGGER_INFO); if (use_audio) { - if (audiodelay >= 0) { - LOGI("Audio-only ALAC streams will be delayed by %d microseconds", audiodelay); - } audio_renderer_init(render_logger, audiosink.c_str()); } else { LOGI("audio_disabled"); From 5f564556c9500bfbf9f49f24c9c948853f4eda7b Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Wed, 8 Feb 2023 01:31:44 -0500 Subject: [PATCH 6/7] add a -sync option to activate server-client sync in audio-only mode --- README.html | 18 ++++++++++++++++-- README.md | 14 +++++++++++++- README.txt | 19 +++++++++++++++++-- renderers/audio_renderer.h | 2 +- renderers/audio_renderer_gstreamer.c | 8 ++++++-- uxplay.1 | 3 ++- uxplay.cpp | 7 +++++-- 7 files changed, 60 insertions(+), 11 deletions(-) diff --git a/README.html b/README.html index ca77351..d007c0a 100644 --- a/README.html +++ b/README.html @@ -394,6 +394,15 @@ client drops the connection; since UxPlay-1.58, the option -nohold modifies this behavior so that when a new client requests a connection, it removes the current client and takes over.

    +

  • In Audio-Only mode, use the -sync option to +synchronize audio on the server with video on the client. This +introduces a delay that the client adds to account for latency. There is +an option -al x that sets the audio latency x that +the server reports to the client that in principle might modify the +delay a little (this is not clear). Here (non-negative decimal) +x is given in seconds, default is 0.25. Without the +-sync option, there is no audio delay, but the client’s +video lags behind the server’s audio.

  • Since UxPlay-1.54, you can display the accompanying “Cover Art” from sources like Apple Music in Audio-Only (ALAC) mode: run “uxplay -ca <name> &” in the background, then run @@ -680,6 +689,12 @@ the mirror display (X11) window.

    -nh Do not append “@_hostname_” at the end of the AirPlay server name.

    +

    -sync (In Audio-Only (ALAC)) mode: this option +synchronizes audio on the server with video on the client, but causes +the client to add a delay to account for latency, so pausing the stream +will not take effect immediately. This can be mitigated by using the +-al audio latency setting to change the latency (default +0.25 secs) that the server reports to the cient.

    -s wxh (e.g. -s 1920x1080 , which is the default ) sets the display resolution (width and height, in pixels). (This may be a request made to the AirPlay client, and perhaps will not be the final @@ -780,8 +795,7 @@ in (decimal) seconds in Audio-only (ALAC), that is reported to the client. Values in the range [0.0, 10.0] seconds are allowed, and will be converted to a whole number of microseconds. Default is 0.25 sec (250000 usec). (This replaces the -ao option introduced in v1.62, -as a workaround for a problem that is now fixed: it is not clear if -changing the value of x produces any effects, however.)

    +as a workaround for a problem that is now fixed).

    -ca filename provides a file (where filename can include a full path) used for output of “cover art” (from Apple Music, etc.,) in audio-only ALAC mode. This diff --git a/README.md b/README.md index ff0b3a8..d4c388d 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,12 @@ per second. (You can see what framerate is actually streaming by using -vs fpsdi its current client until that client drops the connection; since UxPlay-1.58, the option `-nohold` modifies this behavior so that when a new client requests a connection, it removes the current client and takes over. +* In Audio-Only mode, use the `-sync` option to synchronize audio on the server with video +on the client. This introduces a delay that the client adds to account for latency. There is an +option `-al x` that sets the audio latency _x_ that the server reports to the client that in principle +might modify the delay a little (this is not clear). Here (non-negative decimal) _x_ is given in seconds, default +is 0.25. _Without the `-sync` option, there is no audio delay, but the client's video lags behind the server's audio._ + * Since UxPlay-1.54, you can display the accompanying "Cover Art" from sources like Apple Music in Audio-Only (ALAC) mode: run "`uxplay -ca &`" in the background, then run a image viewer with an autoreload feature: an example is "feh": run "``feh -R 1 ``" @@ -566,6 +572,12 @@ Options: **-nh** Do not append "@_hostname_" at the end of the AirPlay server name. +**-sync** (In Audio-Only (ALAC)) mode: this option synchronizes audio on the server with video on the client, + but causes the client to add a delay to account for latency, so pausing the stream will not take effect + immediately. This can be mitigated by using the `-al` audio latency setting to change the latency (default 0.25 secs) + that the server reports to the cient. + + **-s wxh** (e.g. -s 1920x1080 , which is the default ) sets the display resolution (width and height, in pixels). (This may be a request made to the AirPlay client, and perhaps will not @@ -660,7 +672,7 @@ which will not work if a firewall is running. **-al _x_** specifies an audio latency _x_ in (decimal) seconds in Audio-only (ALAC), that is reported to the client. Values in the range [0.0, 10.0] seconds are allowed, and will be converted to a whole number of microseconds. Default is 0.25 sec (250000 usec). (This replaces the `-ao` option introduced in v1.62, as a workaround for a problem that - is now fixed: it is not clear if changing the value of _x_ produces any effects, however.) + is now fixed). **-ca _filename_** provides a file (where _filename_ can include a full path) used for output of "cover art" (from Apple Music, _etc._,) in audio-only ALAC mode. This file is overwritten with the latest cover art as diff --git a/README.txt b/README.txt index 6bf3857..9399e60 100644 --- a/README.txt +++ b/README.txt @@ -397,6 +397,15 @@ for help with this or other problems. modifies this behavior so that when a new client requests a connection, it removes the current client and takes over. +- In Audio-Only mode, use the `-sync` option to synchronize audio on + the server with video on the client. This introduces a delay that + the client adds to account for latency. There is an option `-al x` + that sets the audio latency *x* that the server reports to the + client that in principle might modify the delay a little (this is + not clear). Here (non-negative decimal) *x* is given in seconds, + default is 0.25. *Without the `-sync` option, there is no audio + delay, but the client's video lags behind the server's audio.* + - Since UxPlay-1.54, you can display the accompanying "Cover Art" from sources like Apple Music in Audio-Only (ALAC) mode: run "`uxplay -ca &`" in the background, then run a image viewer @@ -691,6 +700,13 @@ will also now be the name shown above the mirror display (X11) window. **-nh** Do not append "@_hostname_" at the end of the AirPlay server name. +**-sync** (In Audio-Only (ALAC)) mode: this option synchronizes audio on +the server with video on the client, but causes the client to add a +delay to account for latency, so pausing the stream will not take effect +immediately. This can be mitigated by using the `-al` audio latency +setting to change the latency (default 0.25 secs) that the server +reports to the cient. + **-s wxh** (e.g. -s 1920x1080 , which is the default ) sets the display resolution (width and height, in pixels). (This may be a request made to the AirPlay client, and perhaps will not be the final resolution you @@ -801,8 +817,7 @@ Audio-only (ALAC), that is reported to the client. Values in the range \[0.0, 10.0\] seconds are allowed, and will be converted to a whole number of microseconds. Default is 0.25 sec (250000 usec). (This replaces the `-ao` option introduced in v1.62, as a workaround for a -problem that is now fixed: it is not clear if changing the value of *x* -produces any effects, however.) +problem that is now fixed). **-ca *filename*** provides a file (where *filename* can include a full path) used for output of "cover art" (from Apple Music, *etc.*,) in diff --git a/renderers/audio_renderer.h b/renderers/audio_renderer.h index adeea75..6481eb1 100644 --- a/renderers/audio_renderer.h +++ b/renderers/audio_renderer.h @@ -30,7 +30,7 @@ extern "C" { #include "../lib/logger.h" bool gstreamer_init(); -void audio_renderer_init(logger_t *logger, const char* audiosink); +void audio_renderer_init(logger_t *logger, const char* audiosink, const bool *audio_sync); void audio_renderer_start(unsigned char* compression_type); void audio_renderer_stop(); void audio_renderer_render_buffer(unsigned char* data, int *data_len, unsigned short *seqnum, uint64_t *ntp_time); diff --git a/renderers/audio_renderer_gstreamer.c b/renderers/audio_renderer_gstreamer.c index 239bc16..d1760dd 100644 --- a/renderers/audio_renderer_gstreamer.c +++ b/renderers/audio_renderer_gstreamer.c @@ -82,7 +82,7 @@ static GstClockTime gst_audio_pipeline_base_time = GST_CLOCK_TIME_NONE; static logger_t *logger = NULL; const char * format[NFORMATS]; -void audio_renderer_init(logger_t *render_logger, const char* audiosink) { +void audio_renderer_init(logger_t *render_logger, const char* audiosink, const bool* audio_sync) { GError *error = NULL; GstCaps *caps = NULL; GstClock *clock = gst_system_clock_obtain(); @@ -114,7 +114,11 @@ void audio_renderer_init(logger_t *render_logger, const char* audiosink) { g_string_append (launch, audiosink); switch(i) { case 1: /*ALAC*/ - g_string_append (launch, " sync=true"); + if (*audio_sync) { + g_string_append (launch, " sync=true"); + } else { + g_string_append (launch, " sync=false"); + } break; default: g_string_append (launch, " sync=false"); diff --git a/uxplay.1 b/uxplay.1 index f91280f..60a8125 100644 --- a/uxplay.1 +++ b/uxplay.1 @@ -13,7 +13,8 @@ UxPlay 1.63: An open\-source AirPlay mirroring (+ audio streaming) server. .TP \fB\-nh\fR Do \fBNOT\fR append "@\fIhostname\fR" at end of the AirPlay server name .TP -.B +\fB\-sync\fR (In Audio-Only mode) sync audio on server with video on client. +.TP \fB\-s\fR wxh[@r]Set display resolution [refresh_rate] default 1920x1080[@60] .TP \fB\-o\fR Set display "overscanned" mode on (not usually needed) diff --git a/uxplay.cpp b/uxplay.cpp index e97a4fd..a63eb9f 100644 --- a/uxplay.cpp +++ b/uxplay.cpp @@ -65,7 +65,7 @@ static std::string server_name = DEFAULT_NAME; static dnssd_t *dnssd = NULL; static raop_t *raop = NULL; static logger_t *render_logger = NULL; - +static bool audio_sync = false; static bool relaunch_video = false; static bool reset_loop = false; static unsigned int open_connections= 0; @@ -359,6 +359,7 @@ static void print_info (char *name) { printf("Options:\n"); printf("-n name Specify the network name of the AirPlay server\n"); printf("-nh Do not add \"@hostname\" at the end of the AirPlay server name\n"); + printf("-sync (In Audio-Only mode) sync audio on server with video on client\n"); printf("-s wxh[@r]Set display resolution [refresh_rate] default 1920x1080[@60]\n"); printf("-o Set display \"overscanned\" mode on (not usually needed)\n"); printf("-fs Full-screen (only works with X11, Wayland and VAAPI)\n"); @@ -552,6 +553,8 @@ static void parse_arguments (int argc, char *argv[]) { server_name = std::string(argv[++i]); } else if (arg == "-nh") { do_append_hostname = false; + } else if (arg == "-sync") { + audio_sync = true; } else if (arg == "-s") { if (!option_has_value(i, argc, argv[i], argv[i+1])) exit(1); std::string value(argv[++i]); @@ -1280,7 +1283,7 @@ int main (int argc, char *argv[]) { logger_set_level(render_logger, debug_log ? LOGGER_DEBUG : LOGGER_INFO); if (use_audio) { - audio_renderer_init(render_logger, audiosink.c_str()); + audio_renderer_init(render_logger, audiosink.c_str(), &audio_sync); } else { LOGI("audio_disabled"); } From 995a4bded8060e9b154695dca31add95a8011761 Mon Sep 17 00:00:00 2001 From: "F. Duncanh" Date: Thu, 9 Feb 2023 03:06:36 -0500 Subject: [PATCH 7/7] improved debug output for timings --- lib/raop_ntp.c | 17 ++++++++++------- lib/raop_rtp.c | 6 +++--- lib/raop_rtp_mirror.c | 32 ++++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lib/raop_ntp.c b/lib/raop_ntp.c index 6318b82..5823e54 100644 --- a/lib/raop_ntp.c +++ b/lib/raop_ntp.c @@ -285,8 +285,10 @@ raop_ntp_thread(void *arg) byteutils_put_ntp_timestamp(request, 24, send_time); int send_len = sendto(raop_ntp->tsock, (char *)request, sizeof(request), 0, (struct sockaddr *) &raop_ntp->remote_saddr, raop_ntp->remote_saddr_len); - logger_log(raop_ntp->logger, LOGGER_DEBUG, "\nraop_ntp send_len = %d, now = %8.6f", send_len, - (double) send_time / SECOND_IN_NSECS); + char *str = utils_data_to_string(request, send_len, 16); + logger_log(raop_ntp->logger, LOGGER_DEBUG, "\nraop_ntp send time type_t=%d send_len = %d, now = %8.6f\n%s", + request[1] &~0x80, send_len, (double) send_time / SECOND_IN_NSECS, str); + free(str); if (send_len < 0) { logger_log(raop_ntp->logger, LOGGER_ERR, "raop_ntp error sending request"); } else { @@ -308,10 +310,6 @@ raop_ntp_thread(void *arg) //local time of the server when the NTP response packet returns int64_t t3 = (int64_t) raop_ntp_get_local_time(raop_ntp); timeout_counter = 0; - char *str = utils_data_to_string(response, response_len, 16); - logger_log(raop_ntp->logger, LOGGER_DEBUG, "raop_ntp receive time type_t=%d packetlen = %d\n%s", - response[1] &~0x80, response_len, str); - free(str); // Local time of the server when the NTP request packet leaves the server int64_t t0 = (int64_t) byteutils_get_ntp_timestamp(response, 8); @@ -322,7 +320,12 @@ raop_ntp_thread(void *arg) // Local time of the client when the response message leaves the client int64_t t2 = (int64_t) byteutils_get_ntp_timestamp(response, 24); - // The iOS client device sends its time in seconds relative to an arbitrary Epoch (the last boot). + char *str = utils_data_to_string(response, response_len, 16); + logger_log(raop_ntp->logger, LOGGER_DEBUG, "raop_ntp receive time type_t=%d packetlen = %d, now = %8.6f t1 = %8.6f, t2 = %8.6f\n%s", + response[1] &~0x80, response_len, (double) t3 / SECOND_IN_NSECS, (double) t1 / SECOND_IN_NSECS, (double) t2 / SECOND_IN_NSECS, str); + free(str); + + // The iOS client device sends its time in seconds relative to an arbitrary Epoch (the last boot). // For a little bonus confusion, they add SECONDS_FROM_1900_TO_1970. // This means we have to expect some rather huge offset, but its growth or shrink over time should be small. diff --git a/lib/raop_rtp.c b/lib/raop_rtp.c index 2d7979e..1789b99 100644 --- a/lib/raop_rtp.c +++ b/lib/raop_rtp.c @@ -573,9 +573,9 @@ raop_rtp_thread_udp(void *arg) uint64_t sync_ntp_local = raop_ntp_convert_remote_time(raop_rtp->ntp, sync_ntp_remote); char *str = utils_data_to_string(packet, packetlen, 20); logger_log(raop_rtp->logger, LOGGER_DEBUG, - "raop_rtp sync: client ntp=%8.6f, ntp = %8.6f, ntp_start_time %8.6f, sync_rtp=%u\n%s", - ((double) sync_ntp_remote) / SEC, ((double)sync_ntp_local) / SEC, - ((double) raop_rtp->ntp_start_time) / SEC, sync_rtp, str); + "raop_rtp sync: client ntp=%8.6f, ntp = %8.6f, ntp_start_time %8.6f\nts_client = %8.6f sync_rtp=%u\n%s", + (double) sync_ntp_remote / SEC, (double) sync_ntp_local / SEC, + (double) raop_rtp->ntp_start_time / SEC, (double) sync_ntp_remote / SEC, sync_rtp, str); free(str); raop_rtp_sync_clock(raop_rtp, &sync_ntp_remote, &sync_rtp64); } else { diff --git a/lib/raop_rtp_mirror.c b/lib/raop_rtp_mirror.c index 93ac87f..ad725ba 100644 --- a/lib/raop_rtp_mirror.c +++ b/lib/raop_rtp_mirror.c @@ -199,6 +199,8 @@ raop_rtp_mirror_thread(void *arg) bool conn_reset = false; uint64_t ntp_timestamp_nal = 0; uint64_t ntp_timestamp_raw = 0; + uint64_t ntp_timestamp_remote = 0; + uint64_t ntp_timestamp; unsigned char nal_start_code[4] = { 0x00, 0x00, 0x00, 0x01 }; #ifdef DUMP_H264 @@ -309,7 +311,15 @@ raop_rtp_mirror_thread(void *arg) /*packet[0:3] contains the payload size */ int payload_size = byteutils_get_int(packet, 0); - + char packet_description[13] = {0}; + char *p = packet_description; + for (int i = 4; i < 8; i++) { + sprintf(p, "%2.2x ", (unsigned int) packet[i]); + p += 3; + } + ntp_timestamp_raw = byteutils_get_long(packet, 8); + ntp_timestamp_remote = raop_ntp_timestamp_to_nano_seconds(ntp_timestamp_raw, false); + /* packet[4] appears to have one of three possible values: * * 0x00 : encrypted packet * * 0x01 : unencrypted packet with a SPS and a PPS NAL, sent initially, and also when * @@ -361,14 +371,12 @@ raop_rtp_mirror_thread(void *arg) // so no additional clock syncing needed. The only thing odd here is that the video // ntp time stamps don't include the SECONDS_FROM_1900_TO_1970, so it's really just // counting nano seconds since last boot. - ntp_timestamp_raw = byteutils_get_long(packet, 8); - uint64_t ntp_timestamp_remote = raop_ntp_timestamp_to_nano_seconds(ntp_timestamp_raw, false); - uint64_t ntp_timestamp = raop_ntp_convert_remote_time(raop_rtp_mirror->ntp, ntp_timestamp_remote); + ntp_timestamp = raop_ntp_convert_remote_time(raop_rtp_mirror->ntp, ntp_timestamp_remote); uint64_t ntp_now = raop_ntp_get_local_time(raop_rtp_mirror->ntp); int64_t latency = ((int64_t) ntp_now) - ((int64_t) ntp_timestamp); - logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp video: now = %8.6f, ntp = %8.6f, latency = %8.6f", - ((double) ntp_now) / SEC, ((double) ntp_timestamp) / SEC, ((double) latency) / SEC); + logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp video: now = %8.6f, ntp = %8.6f, latency = %8.6f, ts = %8.6f, %s", + (double) ntp_now / SEC, (double) ntp_timestamp / SEC, (double) latency / SEC, (double) ntp_timestamp_remote / SEC, packet_description); #ifdef DUMP_H264 fwrite(payload, payload_size, 1, file_source); @@ -444,7 +452,7 @@ raop_rtp_mirror_thread(void *arg) if (prepend_sps_pps) { h264_data.data_len += raop_rtp_mirror->sps_pps_len; h264_data.nal_count += 2; - if (ntp_timestamp_raw != ntp_timestamp_nal) { + if (ntp_timestamp_remote != ntp_timestamp_nal) { logger_log(raop_rtp_mirror->logger, LOGGER_WARNING, "raop_rtp_mirror: prepended sps_pps timestamp does not match that of video payload"); } } @@ -454,11 +462,13 @@ raop_rtp_mirror_thread(void *arg) case 0x01: // The information in the payload contains an SPS and a PPS NAL // The sps_pps is not encrypted + logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "\nReceived unencryted codec packet from client: payload_size %d header %s ts_client = %8.6f", + payload_size, packet_description, (double) ntp_timestamp_remote / SEC); if (payload_size == 0) { logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror, discard type 0x01 packet with no payload"); break; } - ntp_timestamp_nal = byteutils_get_long(packet, 8); + ntp_timestamp_nal = ntp_timestamp_remote; float width = byteutils_get_float(packet, 16); float height = byteutils_get_float(packet, 20); float width_source = byteutils_get_float(packet, 40); @@ -538,7 +548,8 @@ raop_rtp_mirror_thread(void *arg) break; case 0x05: - logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "\nReceived video streaming performance info packet from client"); + logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "\nReceived video streaming performance info packet from client: payload_size %d header %s ts_raw = %llu", + payload_size, packet_description, ntp_timestamp_raw); /* payloads with packet[4] = 0x05 have no timestamp, and carry video info from the client as a binary plist * * Sometimes (e.g, when the client has a locked screen), there is a 25kB trailer attached to the packet. * * This 25000 Byte trailer with unidentified content seems to be the same data each time it is sent. */ @@ -567,7 +578,8 @@ raop_rtp_mirror_thread(void *arg) } break; default: - logger_log(raop_rtp_mirror->logger, LOGGER_WARNING, "\nReceived unexpected TCP packet from client, packet[4] = 0x%2.2x", packet[4]); + logger_log(raop_rtp_mirror->logger, LOGGER_WARNING, "\nReceived unexpected TCP packet from client, size %d, %s ts_raw = raw%llu", + payload_size, packet_description, ntp_timestamp_raw); break; }