Commit b637fe3c authored by Andreas Schmidt's avatar Andreas Schmidt

Hardware timestamping support.

parent a45e3753
Pipeline #1547 passed with stages
in 1 minute and 22 seconds
# Predictable Reliable Realtime Transport
## Docker
sudo docker run --rm --name=prrt_recv --net=host prrt receiver 5000 127
sudo docker run --rm --name=prrt_send --net=host prrt sender 127.0.0.1 5000 127
bridge=docker network create --subnet="10.5.1.0/24" prrt | cut -c1-12
ovs-vsctl add-port of-switch br-$bridge
docker run --rm --name=prrt_recv --network="prrt" -v=/opt/prrt:/output --ip=10.5.1.52 --cap-add NET_ADMIN git.nt.uni-saarland.de:4567/larn/prrt:develop receiver 5000 127
docker run --rm --name=prrt_send --network="prrt" -v=/opt/prrt:/output --ip=10.5.1.51 --cap-add NET_ADMIN git.nt.uni-saarland.de:4567/larn/prrt:develop sender 10.50.1.52 5000 127 rate 1mbit
# Predictably Reliable Real-Time (PRRT)
## Hardware Timestamping
* Hardware timestamping can be enabled per socket by calling `PrrtSocket_enable_hardware_timestamping()` and specifying the interface.
* Requirements are a kernel that supports physical timestamping. Instructions can be found
### Compiling a hardware tmestamping enabled kernel on Ubuntu
* A detailed tutorial on kernel compilation can be found [here](https://help.ubuntu.com/community/Kernel/Compile).
* Create and enter a new folder, e.g. in your home directory that will store the newly compiled kernel.
* Run the following commands to download all resources required:
```bash
sudo apt install libncurses5 libncurses5-dev
sudo apt build-dep linux-image-`uname -r`
apt source linux-image-`uname -r`
cd linux-$(uname -r | awk '{split($1,a, "-"); print a[1]}')
```
* Copy the current kernel configuration to the current folder:
```bash
cp -vi /boot/config-`uname -r` .config
```
* Edit the `.config` file and ensure the following line is present (and the option is set to `y`):
```bash
...
CONFIG_NETWORK_PHY_TIMESTAMPING=y
...
```
* Call `make deb-pkg` (ideally with `-j5` or some other number to speed up the compilation by using multiple cores).
* Install the `linux-headers-*.deb` and `linux-image-*.deb` package via `dpkg -i` (where * stands for the respective kernel version).
* Reboot the system.
## Docker
The following shows how to run the PRRT sender and receiver on a host with OpenvSwitch.
```bash
bridge=$(docker network create --subnet="10.5.1.0/24" prrt | cut -c1-12)
ovs-vsctl add-port of-switch br-$bridge
docker run --rm --name=prrt_recv --network="prrt" -v=/opt/prrt:/output --ip=10.5.1.52 --cap-add NET_ADMIN docker.nt.uni-saarland.de/larn/prrt:develop receiver 5000 127
docker run --rm --name=prrt_send --network="prrt" -v=/opt/prrt:/output --ip=10.5.1.51 --cap-add NET_ADMIN docker.nt.uni-saarland.de/larn/prrt:develop sender 10.50.1.52 5000 127 rate 1mbit
```
\ No newline at end of file
#include <sys/time.h>
#include <stddef.h>
#include <stdlib.h>
#include "../util/common.h"
#include "../util/dbg.h"
#include "clock.h"
#include "packet.h"
prrtTimestamp_t PrrtClock_get_current_time_us()
{
......
......@@ -27,8 +27,8 @@ prrtTimestamp_t PrrtClock_get_prrt_time_us(PrrtClock *clock);
bool PrrtClock_update(PrrtClock *clock, prrtTimestamp_t referenceTime, prrtTimedelta_t rtt);
#define diff_abs_ts(timeA, timeB) (prrtTimeDifference_t) abs(((prrtTimeDifference_t) timeA) - ((prrtTimeDifference_t)timeB))
#define PrrtClock_TimespecToPrrtTimestamp(timespec) ((prrtTimestamp_t) (1000000 * timespec.tv_sec + timespec.tv_nsec / 1000))
#endif //PRRT_CLOCK_H
This diff is collapsed.
#include <netdb.h>
#include <stdio.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include "../../defines.h"
#include "../../util/dbg.h"
#include "../../util/common.h"
......@@ -99,7 +100,7 @@ static void handle_data_packet(PrrtSocket *sock_ptr, PrrtPacket *packet, struct
sock_ptr->lastSentTimestamp = dataTimestamp;
PrrtClock_update(&sock_ptr->clock, dataTimestamp, payload->groupRTT_us);
PrrtPacketTimeout* packetTimeout = PrrtPacketTimeout_create(packet);
PrrtPacketTimeout *packetTimeout = PrrtPacketTimeout_create(packet);
check(PrrtPacketTimeoutTable_insert(sock_ptr->packetTimeoutTable, packetTimeout), "Could not insert data packet.");
prrtSequenceNumber_t seqno = packet->sequenceNumber;
......@@ -126,7 +127,8 @@ static void handle_data_packet(PrrtSocket *sock_ptr, PrrtPacket *packet, struct
PrrtBlock *block = PrrtRepairBlockStore_get_block(sock_ptr->repairBlockStore, baseSequenceNumber);
if (block != NULL) {
check(PrrtBlock_insert_data_packet(block, reference), "Inserting failed: %d, %d", baseSequenceNumber, seqno);
check(PrrtBlock_insert_data_packet(block, reference), "Inserting failed: %d, %d", baseSequenceNumber,
seqno);
decode_block(sock_ptr, block);
} else {
if (PrrtDataStore_insert(sock_ptr->dataPacketStore, reference) == false) {
......@@ -177,6 +179,48 @@ static void handle_redundancy_packet(PrrtSocket *socket, PrrtPacket *packet) {
return;
}
void receive_from_socket(const PrrtSocket *socket_ptr, unsigned char *buffer_ptr, ssize_t *received_size,
struct sockaddr_in *remote_ptr, socklen_t *remote_len_ptr, struct timespec *packet_stamp_ptr) {
if(socket_ptr->isHardwareTimestamping) {
struct cmsghdr *cmsg;
struct msghdr msg;
struct iovec entry;
struct {
struct cmsghdr cm;
char control[512];
} control;
memset(&msg, 0, sizeof(msg));
msg.msg_iov = &entry;
msg.msg_iovlen = 1;
entry.iov_base = buffer_ptr;
entry.iov_len = MAX_PAYLOAD_LENGTH;
msg.msg_name = (caddr_t) remote_ptr;
msg.msg_namelen = *remote_len_ptr;
msg.msg_control = &control;
msg.msg_controllen = sizeof(control);
*received_size = recvmsg(socket_ptr->dataSocketFd, &msg, 0);
for(cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
switch(cmsg->cmsg_type) {
case SO_TIMESTAMPNS: {
memcpy(packet_stamp_ptr, (struct timespec*) CMSG_DATA(cmsg), sizeof(struct timespec));
break;
}
default:
PERROR("Unknown Control Msg Type: %d", cmsg->cmsg_type);
break;
}
}
} else {
*received_size = recvfrom(socket_ptr->dataSocketFd, buffer_ptr, MAX_PAYLOAD_LENGTH, 0,
(struct sockaddr *) remote_ptr, remote_len_ptr);
clock_gettime(CLOCK_REALTIME, packet_stamp_ptr);
}
}
void *receive_data_loop(void *ptr) {
ssize_t n;
struct sockaddr_in remote;
......@@ -192,8 +236,12 @@ void *receive_data_loop(void *ptr) {
XlapTimestampPlaceholderInitialize(&tsph2);
XlapTimestampPlaceholderInitialize(&tsph3);
n = recvfrom(sock_ptr->dataSocketFd, buffer, MAX_PAYLOAD_LENGTH, 0, (struct sockaddr *) &remote, &addrlen);
sock_ptr->lastReceivedTimestamp = PrrtClock_get_current_time_us();
struct timespec packet_recv_stamp;
receive_from_socket(sock_ptr, buffer, &n, &remote, &addrlen, &packet_recv_stamp);
debug(DEBUG_HARDSTAMPING, "Packet TS:\t%ld.%09ld; Who? %s", (long) packet_recv_stamp.tv_sec,
packet_recv_stamp.tv_nsec, inet_ntoa(remote.sin_addr));
sock_ptr->lastReceivedTimestamp = PrrtClock_TimespecToPrrtTimestamp(packet_recv_stamp);
XlapTimeStampClock(&tsph1, ts_any_packet, 0, LinkReceive);
XlapTimeStampCycle(&tsph1, ts_any_packet, 0, LinkReceive);
......@@ -209,6 +257,8 @@ void *receive_data_loop(void *ptr) {
prrtPacketType_t packetType = PrrtPacket_type(packet);
debug(DEBUG_DATARECEIVER, "received packet %d:%u", (int) packetType, seqno);
if (packetType == PACKET_TYPE_DATA) {
XlapTimeStampValue(sock_ptr, ts_data_packet, seqno, ChannelReceive, packet_recv_stamp);
XlapTimestampPlaceholderUse(sock_ptr, ts_data_packet, seqno, &tsph1);
XlapTimestampPlaceholderUse(sock_ptr, ts_data_packet, seqno, &tsph2);
XlapTimestampPlaceholderUse(sock_ptr, ts_data_packet, seqno, &tsph3);
......@@ -217,6 +267,8 @@ void *receive_data_loop(void *ptr) {
handle_data_packet(sock_ptr, packet, remote);
XlapTimeStampCycle(sock_ptr, ts_data_packet, seqno, HandlePacketEnd);
} else if (packetType == PACKET_TYPE_REDUNDANCY) {
XlapTimeStampValue(sock_ptr, ts_redundancy_packet, seqno, ChannelReceive, packet_recv_stamp);
XlapTimestampPlaceholderUse(sock_ptr, ts_redundancy_packet, seqno, &tsph1);
XlapTimestampPlaceholderUse(sock_ptr, ts_redundancy_packet, seqno, &tsph2);
XlapTimestampPlaceholderUse(sock_ptr, ts_redundancy_packet, seqno, &tsph3);
......
#include <stdio.h>
#include <netdb.h>
#include <unistd.h>
#include <string.h>
#include "../../defines.h"
#include "../block.h"
......@@ -10,6 +9,66 @@
#include "../../util/common.h"
#include "dataTransmitter.h"
bool send_to_socket(PrrtSocket* sock_ptr, PrrtReceiver *recv, uint8_t* buf, prrtPacketLength_t length, struct timespec *packet_stamp_ptr) {
if(sock_ptr->isHardwareTimestamping) {
struct msghdr msg;
struct iovec iov;
char control[1024];
ssize_t got;
int check = 0;
const int check_max = 9999;
iov.iov_base = buf;
iov.iov_len = length;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_name = recv->ai->ai_addr;
msg.msg_namelen = recv->ai->ai_addrlen;
msg.msg_control = control;
msg.msg_controllen = 0;
sendmsg(sock_ptr->dataSocketFd, &msg, 0);
msg.msg_control = control;
iov.iov_len = MAX_PAYLOAD_LENGTH;
do {
msg.msg_controllen = 1024;
got = recvmsg(sock_ptr->dataSocketFd, &msg, MSG_ERRQUEUE);
} while(got < 0 && errno == EAGAIN && check++ < check_max);
check(!(got < 0 && errno == EAGAIN), "Failed to get stamp. Gave up.");
struct cmsghdr* cmsg;
for(cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
if (cmsg->cmsg_level != SOL_SOCKET) {
continue;
}
switch(cmsg->cmsg_type) {
case SO_TIMESTAMPING: {
// Note: The raw stamp [2] is used, because the others are 0.
struct timespec *hardstamp = &(((struct timespec*) CMSG_DATA(cmsg))[2]); //TODO: This is ugly.
memcpy(packet_stamp_ptr, hardstamp, sizeof(struct timespec));
debug(DEBUG_HARDSTAMPING, "Hardware TS:\t%ld.%09ld", (long) packet_stamp_ptr->tv_sec, (long) packet_stamp_ptr->tv_nsec);
break;
}
default:
debug(DEBUG_DATARECEIVER, "Msgtype: %d", cmsg->cmsg_type);
break;
}
}
} else {
// TODO: [LATENCY] By knowing the time encoding etc. that happens upfront, one could make an adjustment here.
check(sendto(sock_ptr->dataSocketFd, buf, length, 0, recv->ai->ai_addr, recv->ai->ai_addrlen) ==
length, "Sendto failed.");
clock_gettime(CLOCK_REALTIME, packet_stamp_ptr);
debug(DEBUG_HARDSTAMPING, "Software TS:\t%ld.%09ld", (long) packet_stamp_ptr->tv_sec, (long) packet_stamp_ptr->tv_nsec);
}
return true;
error:
PERROR("Sending packet failed.%s", "")
return false;
}
static bool send_packet(PrrtSocket *sock_ptr, PrrtPacket *packet) {
uint8_t buf[MAX_PAYLOAD_LENGTH];
......@@ -31,15 +90,27 @@ static bool send_packet(PrrtSocket *sock_ptr, PrrtPacket *packet) {
case PACKET_TYPE_CHANNEL_FEEDBACK:
default:;
}
uint32_t receiver_count = List_count(sock_ptr->receivers);
struct timespec **timestamps = calloc(receiver_count, sizeof(struct timespec *));
int i = 0;
// SENDING TO ALL RECEIVERS
LIST_FOREACH(sock_ptr->receivers, first, next, cur) {
PrrtReceiver *recv = cur->value;
// TODO: [LATENCY] By knowing the time encoding etc. that happens upfront, one could make an adjustment here.
check(sendto(sock_ptr->dataSocketFd, buf, length, 0, recv->ai->ai_addr, recv->ai->ai_addrlen) ==
length, "Sendto failed.");
timestamps[i] = calloc(sizeof(struct timespec), 1);
send_to_socket(sock_ptr, recv, buf, length, timestamps[i]);
i++;
usleep(1);
}
XlapTimeStampValue(sock_ptr, ts_data_packet, packet->sequenceNumber, ChannelTransmit, *timestamps[i-1]);
// TODO: Make less ugly.
for(int j = 0; j < receiver_count; j++) {
free(timestamps[j]);
}
free(timestamps);
switch (PrrtPacket_type(packet)) {
case PACKET_TYPE_DATA:
XlapTimeStampClock(sock_ptr, ts_data_packet, packet->sequenceNumber, LinkTransmitEnd);
......
This diff is collapsed.
......@@ -17,6 +17,11 @@
#include "clock.h"
#include "../xlap/xlap.h"
#ifndef SIOCSHWTSTAMP
# define SIOCSHWTSTAMP 0x89b0
# define SIOCGHWTSTAMP 0x89b1
#endif
typedef struct prrtSocket {
int dataSocketFd;
......@@ -66,39 +71,44 @@ typedef struct prrtSocket {
pthread_attr_t *receiveFeedbackThreadAttr;
pthread_attr_t *sendDataThreadAttr;
pthread_attr_t *receiveDataThreadAttr;
bool isHardwareTimestamping;
char *interfaceName;
} PrrtSocket;
PrrtSocket *PrrtSocket_create(const bool is_sender, prrtTimedelta_t target_delay_us);
PrrtSocket *PrrtSocket_create(bool is_sender, prrtTimedelta_t target_delay_us);
bool PrrtSocket_enable_hardware_timestamping(PrrtSocket *s, const char * interface_name);
bool PrrtSocket_bind(PrrtSocket *sock_ptr, const char *ipAddress, const uint16_t port);
bool PrrtSocket_bind(PrrtSocket *s, const char *ipAddress, const uint16_t port);
bool PrrtSocket_set_sock_opt(PrrtSocket *sock_ptr, const char *name, const uint32_t value);
bool PrrtSocket_set_sock_opt(PrrtSocket *s, const char *name, const uint32_t value);
uint32_t PrrtSocket_get_sock_opt(PrrtSocket *sock_ptr, const char *name);
uint32_t PrrtSocket_get_sock_opt(PrrtSocket *s, const char *name);
bool PrrtSocket_set_coding_parameters(PrrtSocket *sock_ptr, uint8_t k, uint8_t n);
bool PrrtSocket_set_coding_parameters(PrrtSocket *s, uint8_t k, uint8_t n);
int PrrtSocket_interrupt(PrrtSocket *sock_ptr);
int PrrtSocket_interrupt(PrrtSocket *s);
int PrrtSocket_close(PrrtSocket *sock_ptr);
int PrrtSocket_close(PrrtSocket *s);
int PrrtSocket_connect(PrrtSocket *sock_ptr, const char *host, const uint16_t port);
int PrrtSocket_connect(PrrtSocket *s, const char *host, uint16_t port);
int PrrtSocket_send(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len);
int PrrtSocket_send(PrrtSocket *s, const uint8_t *data, size_t data_len);
int32_t PrrtSocket_recv(PrrtSocket *sock_ptr, void *buf_ptr);
int32_t PrrtSocket_recv(PrrtSocket *s, void *buf_ptr);
int32_t PrrtSocket_recv_sync(PrrtSocket *sock_ptr, void *buf_ptr, prrtTimedelta_t time_window_us);
int32_t PrrtSocket_recv_sync(PrrtSocket *s, void *buf_ptr, prrtTimedelta_t time_window_us);
int32_t PrrtSocket_timedrecv(PrrtSocket *sock_ptr, void *buf_ptr, const uint32_t wait_time);
bool PrrtSocket_cleanup(PrrtSocket *socket);
bool PrrtSocket_cleanup(PrrtSocket *s);
bool PrrtSocket_closing(PrrtSocket *socket);
bool PrrtSocket_closing(PrrtSocket *s);
bool PrrtSocket_uses_thread_pinning(PrrtSocket *socket);
bool PrrtSocket_uses_thread_pinning(PrrtSocket *s);
uint32_t PrrtSocket_get_rtt(PrrtSocket *socket);
uint32_t PrrtSocket_get_rtt(PrrtSocket *s);
#endif // PRRT_SOCKET_H
......@@ -38,6 +38,7 @@ int main(int argc, char *const argv[]) {
#endif
sock = PrrtSocket_create(false, HALF_TIMESTAMP-1);
//PrrtSocket_enable_hardware_timestamping(sock, "enp1s0f0");
check(sock != NULL, "Could not create socket.");
check(PrrtSocket_bind(sock, "0.0.0.0", port), "bind failed");
......
......@@ -32,6 +32,7 @@ int main(int argc, char *const argv[]) {
uint16_t local_port = 6000;
PrrtSocket *socket = PrrtSocket_create(true, 60 * 1000 * 1000);
//PrrtSocket_enable_hardware_timestamping(socket, "enp1s0f0");
check(socket != NULL, "Socket create failed.");
//PrrtSocket_set_coding_parameters(socket, 1, 1); // comment this line to re-enable coding.
......
......@@ -8,13 +8,15 @@
#include <errno.h>
#include <string.h>
#define DEBUG_ALL 0
#define DEBUG_BLOCK 0
#define DEBUG_PACKET 1
#define DEBUG_RECEIVER 1
#define DEBUG_SENDER 0
#define DEBUG_SOCKET 1
#define DEBUG_DATARECEIVER 1
#define DEBUG_ALL 0
#define DEBUG_BLOCK 0
#define DEBUG_PACKET 1
#define DEBUG_RECEIVER 1
#define DEBUG_SENDER 0
#define DEBUG_SOCKET 1
#define DEBUG_DATARECEIVER 1
#define DEBUG_DATATRANSMITTER 1
#define DEBUG_HARDSTAMPING 1
#ifdef DEBUG
#define debug(DOMAIN, M, ...) do { if (DEBUG_ALL||(DOMAIN)) fprintf(stderr, "DEBUG %-20s %s:%d: " M "\n", #DOMAIN + 6, __FILE__, __LINE__, ##__VA_ARGS__); } while (0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment