diff --git a/rtp.c b/rtp.c
index 94de81c2..0e679d66 100644
--- a/rtp.c
+++ b/rtp.c
@@ -382,8 +382,10 @@ void *rtp_control_receiver(void *arg) {
             debug_mutex_lock(&conn->reference_time_mutex, 1000, 0);
 
             if (conn->initial_reference_time == 0) {
-              conn->initial_reference_time = remote_time_of_sync;
-              conn->initial_reference_timestamp = sync_rtp_timestamp;
+              if (conn->packet_count_since_flush > 50) {
+                conn->initial_reference_time = remote_time_of_sync;
+                conn->initial_reference_timestamp = sync_rtp_timestamp;
+              }
             } else {
               uint64_t remote_frame_time_interval =
                   conn->remote_reference_timestamp_time -