.gitignore

@@ -9,3 +9,7 @@ tests/__pycache__/
 MANIFEST
 prrt.cpython*.so
 prrt.so
+.ipynb_checkpoints/
+.idea/
+.venv/
+Pipfile.lock

.gitlab-ci.yml

@@ -22,9 +22,9 @@ build:prrt:
     - which cmake
     - which gcc
     - which g++
-    - pip3 list | grep Cython
-    - pip3 list | grep numpy
-    - CC=gcc-5 CXX=g++-5 cmake . -DPRRT_TESTS=1
+    - pip3 list --format=legacy | grep Cython
+    - pip3 list --format=legacy | grep numpy
+    - CC=gcc-5 CXX=g++-5 cmake .
     - make
 
 build:container:
@@ -38,6 +38,17 @@ build:container:
         - docker push $CI_REGISTRY_IMAGE:$DOCKER_TAG
         - docker rmi $CI_REGISTRY_IMAGE:$DOCKER_TAG
 
+build:container_tcp:
+    stage: build
+    tags:
+        - docker
+    script:
+        - export DOCKER_TAG=$(echo "$CI_BUILD_REF_NAME""_tcp" | sed 's#/#_#' | sed 's#^master$#latest#')
+        - docker build -t $CI_REGISTRY_IMAGE:$DOCKER_TAG --build-arg http_proxy=http://www-proxy.uni-saarland.de:3128 -f docker/Dockerfile_tcp .
+        - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+        - docker push $CI_REGISTRY_IMAGE:$DOCKER_TAG
+        - docker rmi $CI_REGISTRY_IMAGE:$DOCKER_TAG
+
 test:prrt_mem:
   stage: test
   dependencies:
@@ -54,6 +65,9 @@ test:prrt_functional:
   tags:
     - bash
   script:
+    - rm CMakeCache.txt
+    - CC=gcc-5 CXX=g++-5 cmake . -DPRRT_TESTS=1
+    - make
     - exec ./prrtTests
 
 test:prrt_python_bindings:
@@ -79,3 +93,19 @@ deploy:pypi:
     - echo "password=$PYPI_PASSWORD" >> ~/.pypirc
     - python3 setup.py check sdist bdist upload -r on
     - rm -vf ~/.pypirc
+
+deploy:profile:
+  stage: deploy
+  tags:
+    - gprof
+  script:
+    - ls -lahv
+    - rm CMakeCache.txt
+    - CC=gcc-5 CXX=g++-5 cmake . -DGPROF=1
+    - make
+    - bash profiling/profile.sh
+  artifacts:
+    paths:
+    - gprof-send.txt
+    - gprof-recv.txt
+    expire_in: 30 days

.gitmodules

 [submodule "prrt/xlap"]
 	path = prrt/xlap
-	url = ../../as/X-Lap.git
+	url = ../X-Lap.git

CMakeLists.txt

@@ -2,16 +2,22 @@ cmake_minimum_required (VERSION 2.8.11)
 project (PRRT)
 
 option(PRRT_TESTS "Build tests" OFF)
+option(GPROF "Compile with profiler" OFF)
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 
 set(CMAKE_C_FLAGS "-O2 -Wall -std=gnu11 -D_GNU_SOURCE -fPIC" )
+set(CMAKE_C_FLAGS_DEBUG "-O0 -fsanitize=undefined -fsanitize=address -g3" )
 set(CMAKE_CXX_FLAGS "-fstack-protector -fstack-protector-all -Wall -std=gnu++11 -D_GNU_SOURCE" )
-set(CMAKE_CXX_FLAGS_DEBUG "-O2 -Wall -ggdb" )
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -Wall -ggdb -fsanitize=undefined -fsanitize=address -g3" )
 set(CMAKE_CXX_FLAGS_RELEASE "-Os -Wall" )
 
+if(GPROF)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg")
+endif()
 
 find_package (Threads)
 find_library(M_LIB m)
@@ -30,4 +36,4 @@ if(PRRT_TESTS)
 endif()
 
 add_custom_target(perftest COMMAND python3 tests/eval.py)
-add_custom_target(memtest COMMAND bash ./memtest.sh DEPENDS sender receiver)
+add_custom_target(memtest COMMAND bash ./memtest.sh DEPENDS sender receiver)
\ No newline at end of file

MANIFEST.in

-include requirements.txt
-recursive-include src *
 recursive-include prrt *
 include versioneer.py
-include prrt/_version.py
 include _version.py

Pipfile

+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+cython       = "==0.29.13"
+jupyter      = "==1.0.0"
+numpy        = "==1.17.0"
+pandas       = "==0.25.1"
+matplotlib   = "==3.1.1"
+ruamel-yaml  = "==0.16.5"
+scikit-learn = "==0.21.3"
+graphviz     = "==0.12"
+prrt         = {editable = true, path = "./"}

README.md

@@ -4,12 +4,14 @@
 
 ## Features
 
-* Forward Error Correction (FEC) using systematic Vandermonde codes
+* Hybrid error control (FEC + ARQ) using systematic Vandermonde codes
+* Congestion control and pacing using a variant of [BBR](https://groups.google.com/forum/#!forum/bbr-dev)
 * Clock synchronization between sending stack and receiving stack
 * Applications can specify packet-level expiration times
 * Different receive modes for ASAP and time-synchronized operation
 * Passive measurement of propagation delay, bottleneck data rate and packet loss rate
 * Packet-level timing analysis using [X-Lap](http://xlap.larn.systems)
+* Wireshark dissector written in Lua
 * [Hardware timestamping support](https://git.nt.uni-saarland.de/LARN/PRRT/wikis/hardware-timestamping)
 
 ## Installation
@@ -29,9 +31,9 @@ port = int(sys.argv[1])
 s = prrt.PrrtSocket(port=port)
 
 while True:
-    d = s.recv()
+    d = s.recv().decode("utf8")
     if d != "Close":
-        print d
+        print(d)
     else:
         break
 ```
@@ -44,25 +46,27 @@ import prrt
 
 host = sys.argv[1]
 port = int(sys.argv[2])
+localport = int(sys.argv[3])
 
-s = prrt.PrrtSocket(port=port)
+s = prrt.PrrtSocket(port=localport)
 s.connect(host, port)
 
+
 for i in range(10):
-    s.send("Packet {}".format(i))
-s.send("Close")
+    s.send("Packet {}".format(i).encode("utf8"))
+s.send("Close".encode("utf8"))
 ```
 
 Start the receiver by:
 
 ```bash
-python receiver.py 5000
+python3 receiver.py 5000
 ```
 
 In a separate terminal, run:
 
 ```bash
-python sender.py 127.0.0.1 5000
+python3 sender.py 127.0.0.1 5000 6000
 ```
 
 This should generate the following output in the receiver console:
@@ -79,6 +83,20 @@ Packet 9
 * [PRRT Wiki](https://git.nt.uni-saarland.de/LARN/PRRT/wikis)
 * [LARN Project Website](http://larn.systems)
 
+## Citing Us
+
+If you find PRRT useful and incorporate it in your works, we are very happy to hear about it. Please also consider to cite us like this:
+
+```bibtex
+@misc{sic2019prrt,
+      author = {Telecommunications Lab at Saarland Informatics Campus},
+      title = {PRRT: Predictably Reliable Real-time Transport},
+      howpublished={Web page},
+      url = {http://prrt.larn.systems},
+      year = {2019}
+}
+```
+
 ## License
 
 [MIT Licence](LICENSE)

dissect/prrt.lua

@@ -21,20 +21,23 @@ local pf_idx = ProtoField.uint8("prrt.index", "Index")
 local pf_seqN = ProtoField.uint16("prrt.sequenceNumber", "Sequence Number")
 
 local pf_data = ProtoField.new("Data", "prrt.data", ftypes.BYTES, base.NONE)
-local pf_data_length = ProtoField.uint32("prrt.data.length", "Length")
 local pf_data_timestamp = ProtoField.uint32("prrt.data.timestamp", "Timestamp")
-local pf_data_groupRTprop = ProtoField.uint32("prrt.data.grouprtprop", "Group RTprop")
+local pf_data_btl_pace = ProtoField.uint32("prrt.data.btl_pace", "Bottleneck Pace")
+local pf_data_length = ProtoField.uint32("prrt.data.length", "Length")
+local pf_data_RTprop = ProtoField.uint32("prrt.data.RTprop", "RTprop")
 local pf_data_packettimeout = ProtoField.uint32("prrt.data.packettimeout", "Packet Timeout")
+local pf_data_btlDatarate = ProtoField.uint32("prrt.data.btl_datarate", "Bottleneck Datarate")
 
 local pf_red = ProtoField.new("Redundancy", "prrt.redundancy", ftypes.BYTES, base.NONE)
-local pf_red_baseSeqN = ProtoField.uint16("prrt.redundancy.baseSequenceNumber", "Base Sequence Number", base.DEC)
 local pf_red_timestamp = ProtoField.uint32("prrt.redundancy.timestamp", "Timestamp")
+local pf_red_btl_pace = ProtoField.uint32("prrt.redundancy.btl_pace", "Bottleneck Pace")
+local pf_red_baseSeqN = ProtoField.uint16("prrt.redundancy.baseSequenceNumber", "Base Sequence Number", base.DEC)
 local pf_red_n = ProtoField.uint8("prrt.redundancy.n", "n")
 local pf_red_k = ProtoField.uint8("prrt.redundancy.k", "k")
 
 local pf_fb = ProtoField.new("Feedback", "prrt.feedback", ftypes.BYTES, base.NONE)
-local pf_fb_groupRTT = ProtoField.uint32("prrt.feedback.groupRTT", "Group RTT")
 local pf_fb_ftt = ProtoField.uint32("prrt.feedback.FTT", "FTT")
+local pf_fb_btlPace = ProtoField.uint32("prrt.feedback.btlPace", "Bottleneck pace")
 local pf_fb_erasurecount = ProtoField.uint16("prrt.feedback.erasureCount", "Erasure count")
 local pf_fb_packetcount = ProtoField.uint16("prrt.feedback.packetCount", "Packet count")
 local pf_fb_gaplength = ProtoField.uint16("prrt.feedback.gapLength", "Gap length")
@@ -52,20 +55,23 @@ prrt_proto.fields = {
     pf_seqN,
 
     pf_data,
-    pf_data_length,
     pf_data_timestamp,
-    pf_data_groupRTprop,
+    pf_data_btl_pace,
+    pf_data_length,
+    pf_data_RTprop,
     pf_data_packettimeout,
+    pf_data_btlDatarate,
 
     pf_red,
-    pf_red_baseSeqN,
     pf_red_timestamp,
+    pf_red_btl_pace,
+    pf_red_baseSeqN,
     pf_red_n,
     pf_red_k,
 
     pf_fb,
-    pf_fb_groupRTT,
     pf_fb_ftt,
+    pf_fb_btlPace,
     pf_fb_erasurecount,
     pf_fb_packetcount,
     pf_fb_gaplength,
@@ -89,9 +95,16 @@ local ex_type = Field.new("prrt.type")
 local function getType() return ex_type()() end
 local function getTypeName() return prrtPacketTypeNames[getType()] end
 
+local ex_index = Field.new("prrt.index")
+local function getIndex() return ex_index()() end
+
 local ex_data_length = Field.new("prrt.data.length")
 local function getDataLength() return ex_data_length()() end
 
+local ex_red_baseseqno = Field.new("prrt.redundancy.baseSequenceNumber")
+local function getRedBaseSeqNo() return ex_red_baseseqno()() end
+
+
 local ex_red_n = Field.new("prrt.redundancy.n")
 local function getRedN() return ex_red_n()() end
 
@@ -104,32 +117,35 @@ local PRRT_MIN_SIZE = 8
 -- create sub-dissectors for different types
 local function dissect_data(buffer, pinfo, root)
     local tree = root:add(pf_data, buffer:range(0))
-    tree:add(pf_data_length, buffer:range(0,4))
-    tree:add(pf_data_timestamp, buffer:range(4,4))
-    tree:add(pf_data_groupRTprop, buffer:range(8,4))
-    tree:add(pf_data_packettimeout, buffer:range(12,4))
-
-    local label = "DATA Len=" .. getDataLength()
+    tree:add(pf_data_timestamp, buffer:range(0,4))
+    tree:add(pf_data_btl_pace, buffer:range(4,4))
+    tree:add(pf_data_length, buffer:range(8,4))
+    tree:add(pf_data_RTprop, buffer:range(12,4))
+    tree:add(pf_data_packettimeout, buffer:range(16,4))
+    tree:add(pf_data_btlDatarate, buffer:range(20,4))
+
+    local label = "[D] Idx=" .. getIndex() .. " Len=" .. getDataLength()
     tree:set_text(label)
     pinfo.cols.info:set(label)
 end
 
 local function dissect_redundancy(buffer, pinfo, root)
     local tree = root:add(pf_red, buffer:range(0))
-    tree:add(pf_red_baseSeqN, buffer:range(0,2))
-    tree:add(pf_red_timestamp, buffer:range(2,4))
-    tree:add(pf_red_n, buffer:range(6,1))
-    tree:add(pf_red_k, buffer:range(7,1))
+    tree:add(pf_red_timestamp, buffer:range(0,4))
+    tree:add(pf_red_btl_pace, buffer:range(4,4))
+    tree:add(pf_red_baseSeqN, buffer:range(8,2))
+    tree:add(pf_red_n, buffer:range(10,1))
+    tree:add(pf_red_k, buffer:range(11,1))
 
-    local label = "REDUNDANCY n=" .. getRedN() .. " k=" .. getRedK()
+    local label = "[R] Idx=" .. getIndex() .. " b=" .. getRedBaseSeqNo() .. " n=" .. getRedN() .. " k=" .. getRedK()
     tree:set_text(label)
     pinfo.cols.info:set(label)
 end
 
 local function dissect_feedback(buffer, pinfo, root)
     local tree = root:add(pf_fb, buffer:range(0))
-    tree:add(pf_fb_groupRTT, buffer:range(0,4))
-    tree:add(pf_fb_ftt, buffer:range(4,4))
+    tree:add(pf_fb_ftt, buffer:range(0,4))
+    tree:add(pf_fb_btlPace, buffer:range(4,4))
     tree:add(pf_fb_erasurecount, buffer:range(8,2))
     tree:add(pf_fb_packetcount, buffer:range(10,2))
     tree:add(pf_fb_gaplength, buffer:range(12,2))
@@ -139,7 +155,7 @@ local function dissect_feedback(buffer, pinfo, root)
     tree:add(pf_fb_acktype, buffer:range(20,1))
     tree:add(pf_fb_ackSeqN, buffer:range(21, 2))
 
-    local label = "FEEDBACK"
+    local label = "[F]"
     tree:set_text(label)
     pinfo.cols.info:set(label)
 end
@@ -198,4 +214,3 @@ function prrt_proto.prefs_changed()
         end
     end
 end
-

dissect/wireshark-1.4.6/packet-prrt.c

@@ -87,7 +87,7 @@ static int hf_prrt_payload = -1;        /* payload with arbitrary length, depend
 static int hf_prrt_receiver_addr = -1;  /* 32 bits IP address of the receiver */
 static int hf_prrt_rtt_probe = -1;      /* 32 bits timestamp of most recent data packet plus delay since reception of this packet. */
 static int hf_prrt_plr = -1;            /* 32 bits packet loss rate measured at the receiver. */
-static int hf_prrt_bw_est = -1;         /* 32 bits bandwidth estimated at the receiver. */
+static int hf_prrt_datarate_est = -1;         /* 32 bits datarate estimated at the receiver. */
 /* The following two fields are the payload of PRRT feedback packet (packet type: 3). */
 static int hf_prrt_packet_no = -1;      /* 16 bits packet no of the first lost packet. */
 static int hf_prrt_retr_round = -1;     /* 16 bits retransmission round of the first lost. (first transmission is 0.) */
@@ -323,7 +323,7 @@ dissect_prrt(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
       offset += 4;
       proto_tree_add_item(prrt_tree, hf_prrt_plr, tvb, offset, 4, FALSE);
       offset += 4;
-      proto_tree_add_item(prrt_tree, hf_prrt_bw_est, tvb, offset, 4, FALSE);
+      proto_tree_add_item(prrt_tree, hf_prrt_datarate_est, tvb, offset, 4, FALSE);
       offset += 4;
 
       if (len == 24) {
@@ -364,7 +364,7 @@ dissect_prrt(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
       offset += 4;
       proto_tree_add_item(prrt_tree, hf_prrt_plr, tvb, offset, 4, FALSE);
       offset += 4;
-      proto_tree_add_item(prrt_tree, hf_prrt_bw_est, tvb, offset, 4, FALSE);
+      proto_tree_add_item(prrt_tree, hf_prrt_datarate_est, tvb, offset, 4, FALSE);
       offset += 4;
 
       if (len == 28) {
@@ -484,10 +484,10 @@ proto_register_prrt(void)
       FT_UINT32, BASE_DEC, NULL, 0x0,
       "The packet loss rate measured at the receiver, stored in general PRRT feedback header", HFILL }
     },
-    { &hf_prrt_bw_est,
-      { "Estimated Bandwidth", "prrt.feedback.bw",
+    { &hf_prrt_datarate_est,
+      { "Estimated Datarate", "prrt.feedback.datarate",
       FT_UINT32, BASE_DEC, NULL, 0x0,
-      "The bandwidth estimated at the receiver, stored in general PRRT feedback header", HFILL }
+      "The datarate estimated at the receiver, stored in general PRRT feedback header", HFILL }
     },
     { &hf_prrt_packet_no,
       { "Packet No", "prrt.feedback.packetno",

dissect/wireshark-1.6.1/packet-prrt.c

@@ -92,7 +92,7 @@ static int hf_prrt_payload = -1;        /* payload with arbitrary length, depend
 static int hf_prrt_receiver_addr = -1;  /* 32 bits IP address of the receiver */
 static int hf_prrt_rtt_probe = -1;      /* 32 bits timestamp of most recent data packet plus delay since reception of this packet. */
 static int hf_prrt_plr = -1;            /* 32 bits packet loss rate measured at the receiver. */
-static int hf_prrt_bw_est = -1;         /* 32 bits bandwidth estimated at the receiver. */
+static int hf_prrt_datarate_est = -1;         /* 32 bits datarate estimated at the receiver. */
 /* The following two fields are the payload of PRRT feedback packet (packet type: 3). */
 static int hf_prrt_packet_no = -1;      /* 16 bits packet no of the first lost packet. */
 static int hf_prrt_retr_round = -1;     /* 16 bits retransmission round of the first lost. (first transmission is 0.) */
@@ -361,7 +361,7 @@ dissect_prrt(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
       offset += 4;
       proto_tree_add_item(prrt_tree, hf_prrt_plr, tvb, offset, 4, FALSE);
       offset += 4;
-      proto_tree_add_item(prrt_tree, hf_prrt_bw_est, tvb, offset, 4, FALSE);
+      proto_tree_add_item(prrt_tree, hf_prrt_datarate_est, tvb, offset, 4, FALSE);
       offset += 4;
 
       if (len == 24) {
@@ -402,7 +402,7 @@ dissect_prrt(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
       offset += 4;
       proto_tree_add_item(prrt_tree, hf_prrt_plr, tvb, offset, 4, FALSE);
       offset += 4;
-      proto_tree_add_item(prrt_tree, hf_prrt_bw_est, tvb, offset, 4, FALSE);
+      proto_tree_add_item(prrt_tree, hf_prrt_datarate_est, tvb, offset, 4, FALSE);
       offset += 4;
 
       if (len == 28) {
@@ -524,10 +524,10 @@ proto_register_prrt(void)
       FT_UINT32, BASE_DEC, NULL, 0x0,
       "The packet loss rate measured at the receiver, stored in general PRRT feedback header", HFILL }
     },
-    { &hf_prrt_bw_est,
-      { "Estimated Bandwidth", "prrt.feedback.bw",
+    { &hf_prrt_datarate_est,
+      { "Estimated Datarate", "prrt.feedback.datarate",
       FT_UINT32, BASE_DEC, NULL, 0x0,
-      "The bandwidth estimated at the receiver, stored in general PRRT feedback header", HFILL }
+      "The datarate estimated at the receiver, stored in general PRRT feedback header", HFILL }
     },
     { &hf_prrt_packet_no,
       { "Packet No", "prrt.feedback.packetno",

dissect/wireshark-2.0.0/packet-prrt.c

@@ -74,7 +74,7 @@ static int hf_prrt_gap = -1;
 static int hf_prrt_ngap = -1;
 static int hf_prrt_burst = -1;
 static int hf_prrt_nburst = -1;
-static int hf_prrt_bw_est = -1;
+static int hf_prrt_datarate_est = -1;
 static int hf_prrt_buf_fb = -1;
 
 /* Payload */
@@ -206,7 +206,7 @@ static void dissect_prrt(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
                         offset += 2;
                         proto_tree_add_item(prrt_tree, hf_prrt_nburst, tvb, offset, 2, FALSE);
                         offset += 2;
-                        proto_tree_add_item(prrt_tree, hf_prrt_bw_est, tvb, offset, 4, FALSE);
+                        proto_tree_add_item(prrt_tree, hf_prrt_datarate_est, tvb, offset, 4, FALSE);
                         offset += 4;
                         proto_tree_add_item(prrt_tree, hf_prrt_buf_fb, tvb, offset, 4, FALSE);
                 } else if(packet_type == 4) {
@@ -236,7 +236,7 @@ static void dissect_prrt(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
                         offset += 2;
                         proto_tree_add_item(prrt_tree, hf_prrt_nburst, tvb, offset, 2, FALSE);
                         offset += 2;
-                        proto_tree_add_item(prrt_tree, hf_prrt_bw_est, tvb, offset, 4, FALSE);
+                        proto_tree_add_item(prrt_tree, hf_prrt_datarate_est, tvb, offset, 4, FALSE);
                         offset += 4;
                         proto_tree_add_item(prrt_tree, hf_prrt_buf_fb, tvb, offset, 4, FALSE);
                 } else {
@@ -292,8 +292,8 @@ void proto_register_prrt(void)
                   { "Aggregated Burst Length", "prrt.feedback.burst", FT_UINT16, BASE_DEC, NULL, 0x0, "", HFILL }},
                 { &hf_prrt_nburst,
                   { "Burst Count", "prrt.feedback.nburst", FT_UINT16, BASE_DEC, NULL, 0x0, "", HFILL }},
-                { &hf_prrt_bw_est,
-                  { "Estimated Bandwidth", "prrt.feedback.bw", FT_UINT32, BASE_DEC, NULL, 0x0, "The bandwidth estimated at the receiver, stored in general PRRT feedback header", HFILL }},
+                { &hf_prrt_datarate_est,
+                  { "Estimated Datarate", "prrt.feedback.datarate", FT_UINT32, BASE_DEC, NULL, 0x0, "The datarate estimated at the receiver, stored in general PRRT feedback header", HFILL }},
                 { &hf_prrt_buf_fb,
                   { "Buffer Feedback", "prrt.feedback.buf_fb", FT_UINT32, BASE_DEC, NULL, 0x0, "", HFILL }},
                 { &hf_prrt_payload,

docker/Dockerfile

-FROM gcc:5
+FROM gcc:8
 
 MAINTAINER Andreas Schmidt <schmidt@nt.uni-saarland.de>
 
-RUN apt-get update \
-    && apt-get upgrade -y \
-    && apt-get install -y cmake
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install --yes --force-yes \
+    bc \
+    cmake \
+    iperf3 \
+    psmisc \
+    traceroute \
+    tshark
 
 COPY CMakeLists.txt /prrt/
 COPY prrt /prrt/prrt
 COPY tests /prrt/tests
 COPY docker/entrypoint.sh /
+COPY docker/sysctl.conf /etc/sysctl.d/01-disable-ipv6.conf
 
 WORKDIR /prrt
 
-RUN cmake . \
+RUN cmake . -DCMAKE_BUILD_TYPE=DEBUG \
     && make
-ENV PATH /prrt/bin:$PATH
-WORKDIR /prrt/bin
+ENV PATH /prrt:$PATH
 
 VOLUME /output
 

docker/Dockerfile_tcp

+FROM gcc:8
+
+MAINTAINER Andreas Schmidt <schmidt@nt.uni-saarland.de>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install --yes --force-yes \
+    bc \
+    cmake \
+    iperf3 \
+    psmisc \
+    traceroute \
+    tshark
+
+COPY CMakeLists.txt /prrt/
+COPY prrt /prrt/prrt
+COPY tests /prrt/tests
+COPY docker/entrypoint.sh /
+
+WORKDIR /prrt
+
+RUN cmake -DTCP=1 -DCMAKE_BUILD_TYPE=DEBUG . \
+    && make
+ENV PATH /prrt:$PATH
+
+VOLUME /output
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/entrypoint.sh

-#!/bin/bash
+#!/bin/sh
 
 dev=eth0
 
 command=$1
 shift
 
-if [[ "$command" == "sender" || "$command" == "receiver" ]]; then
+if [[ "$command" == "sender" || "$command" == "receiver" || "$command" == "time-sender" || "$command" == "time-receiver" ]]; then
     :
 else
     echo "Command should be either sender or receiver."
     exit 0;
 fi
 
+TARGET="127.0.0.1"
+OUTPUT="/dev/null"
 NETEM=()
 PRRT=()
 while [[ $# -gt 0 ]]
@@ -19,7 +21,24 @@ do
 key="$1"
 
 case $key in
-    -t|--target|-p|--port|-r|--rounds)
+    -t|--target)
+    if [[  "$command" == "sender" || "$command" == "time-sender" ]]; then
+    PRRT+=("$1 $2")
+    fi
+    TARGET=("$2")
+    shift
+    shift
+    ;;
+    -w|--wireshark)
+    OUTPUT=("$2")
+    shift
+    shift
+    ;;
+    -T|--threadpinning|-U)
+    PRRT+=("$1")
+    shift
+    ;;
+    -p|--port|-r|--rounds|-s|--size|-R|--rcvbuf|-S|--sndbuf|-o|--output|-a|--appdelay|-j|--appjitter)
     PRRT+=("$1 $2")
     shift
     shift
@@ -34,8 +53,30 @@ done
 PRRT_PARAMS="${PRRT[@]}"
 NETEM_PARAMS="${NETEM[@]}"
 
-echo "Running PRRT with command: \"$command $PRRT_PARAMS\" and link parameters: \"$NETEM_PARAMS\""
+echo "Starting Wireshark."
+tshark -i eth0 -w $OUTPUT.pcap &
+TSHARK_PID=$!
+sleep 2
+
+start=$(date +%s.%N);
+echo "Checking reachability of $TARGET."
+until ping -c1 $TARGET &>/dev/null; do sleep 1; done
+dur=$(echo "$(date +%s.%N) - $start" | bc);
+
+printf "Reachable after %.6f seconds\n" $dur
 
-tc qdisc add dev $dev root netem $NETEM_PARAMS
-/prrt/$command $PRRT_PARAMS -o /output/log.csv
-tc qdisc del dev $dev root
+if [[ "$command" == "sender" || "$command" == "time-sender" ]]; then
+    echo "Delaying sender start."
+else
+    echo "Delaying receiver start."
+fi
+
+start=$(date +%s.%N);
+echo "Running PRRT with command: \"$command $PRRT_PARAMS\" and link parameters: \"$NETEM_PARAMS\""
+trap 'echo "Caught SIGINT."; echo "$(ps -a)"; killall -SIGINT $command' INT
+LOG=$(/prrt/$command $PRRT_PARAMS 2>&1)
+echo "Exit status: $?"
+printf "$LOG\n"
+dur=$(echo "$(date +%s.%N) - $start" | bc);
+printf "Done after %.6f seconds\n" $dur
+kill $TSHARK_PID

docker/sysctl.conf

+net.ipv6.conf.all.disable_ipv6 = 1
+net.ipv6.conf.default.disable_ipv6 = 1
+net.ipv6.conf.lo.disable_ipv6 = 1

eval.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pd.read_csv(\"eval.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

eval.py

+import sys
+sys.path.insert(0, "./build")
+
+import tests.perf as perf
+
+def run_setup_and_report(setup):
+    results = setup.run()
+    print("Setup:\n   ", setup)
+    print("Results:\n    " + str(results).replace("\n","\n    "))
+    results.export()
+
+if __name__ == "__main__":
+    # Packet Count Works: 2^15; Segfault at: 2^16
+
+    # TODO: support multiple tests via proper socket termination
+    setups = [
+        perf.TestSetup(packets=2**17,delay=1,loss=0,reorder=0,duplicate=0)
+    ]
+
+    for setup in setups:
+        run_setup_and_report(setup)

examples/receiver.py

@@ -3,7 +3,7 @@ import prrt
 
 port = int(sys.argv[1])
 
-s = prrt.PrrtSocket(("127.0.0.1", port))
+s = prrt.PrrtSocket(("0.0.0.0", port))
 
 while True:
     d, addr = s.recv()

examples/sender.py

@@ -5,7 +5,7 @@ host = sys.argv[1]
 port = int(sys.argv[2])
 localport = int(sys.argv[3])
 
-s = prrt.PrrtSocket(("127.0.1.1", localport), maximum_payload_size=150)
+s = prrt.PrrtSocket(("0.0.0.0", localport), maximum_payload_size=150)
 s.connect((host, port))
 
 for i in range(10):

profiling/profile.sh

+#!/usr/bin/env bash
+
+which gprof
+
+run_in() {
+    (cd $1; ${@:2})
+}
+
+to="timeout -s INT 30 "
+
+mkdir -p gprof_send gprof_recv
+
+run_in gprof_recv $to ../receiver -p 5000 -r 4095 -o receiver.csv &
+run_in gprof_send $to ../sender -t 127.0.0.1 -p 5000 -r 4095 -o sender.csv
+
+wait
+
+gprof ./receiver gprof_recv/gmon.out > gprof-recv.txt
+gprof ./sender gprof_send/gmon.out > gprof-send.txt
+
+rm -r gprof_send gprof_recv
\ No newline at end of file

prrt/CMakeLists.txt

@@ -10,12 +10,29 @@ if (XLAP)
     add_definitions(-DXLAP)
 endif()
 
+option (TCP "Set time protocol to TCP.")
+
+if (TCP)
+    add_definitions(-DTCP)
+endif()
+
 
 add_subdirectory(proto)
 add_subdirectory(util)
 
 add_executable(sender sender.c)
 add_executable(receiver receiver.c)
+add_executable(time-sender time-sender.c)
+add_executable(time-receiver time-receiver.c)
+
+target_link_libraries(sender LINK_PUBLIC prrt_shared prrtUtil_shared ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(receiver LINK_PUBLIC prrt_shared prrtUtil_shared ${CMAKE_THREAD_LIBS_INIT})
+
+target_link_libraries(time-sender LINK_PUBLIC prrt_shared prrtUtil_shared ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(time-receiver LINK_PUBLIC prrt_shared prrtUtil_shared ${CMAKE_THREAD_LIBS_INIT})
+
+add_executable(refcount refcount.c)
+target_link_libraries(refcount LINK_PUBLIC prrt_shared prrtUtil_shared ${CMAKE_THREAD_LIBS_INIT})
 
-target_link_libraries(sender LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})
-target_link_libraries(receiver LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})
+install(DIRECTORY ./ DESTINATION include/prrt
+        FILES_MATCHING PATTERN "*.h")
\ No newline at end of file

prrt/cprrt.pxd

@@ -119,11 +119,14 @@ cdef extern from "proto/socket.h":
 
     ctypedef prrtSocket PrrtSocket
 
-    cdef PrrtSocket* PrrtSocket_create(const uint32_t mtu, const uint32_t target_delay)
-    bint PrrtSocket_bind(PrrtSocket *sock_ptr, const_char *ipAddress, const uint16_t port)
+    cdef PrrtSocket* PrrtSocket_create(const uint32_t maximum_payload_size, const uint32_t target_delay)
+    int PrrtSocket_bind(PrrtSocket *sock_ptr, const_char *ipAddress, const uint16_t port)
+    bint PrrtSocket_enable_hardware_timestamping(PrrtSocket *s, const_char *interface_name)
+    
     int PrrtSocket_close(const PrrtSocket *sock_ptr)
     int PrrtSocket_connect(PrrtSocket *sock_ptr, const_char *host, const uint16_t port)
-    int PrrtSocket_send(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len)
+    int PrrtSocket_send_sync(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len)
+    int PrrtSocket_send_async(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len)
 
     int32_t PrrtSocket_recv(PrrtSocket *sock_ptr, void *buf_ptr, sockaddr* addr) nogil
     int32_t PrrtSocket_receive_asap(PrrtSocket *s, void *buf_ptr, sockaddr* addr) nogil
@@ -144,11 +147,23 @@ cdef extern from "proto/socket.h":
     uint32_t PrrtSocket_get_rtprop_fwd(PrrtSocket *socket)
     float PrrtSocket_get_plr_fwd(PrrtSocket *socket)
     uint32_t PrrtSocket_get_delivery_rate_fwd(PrrtSocket *socket)
-
-    uint32_t PrrtSocket_get_btlbw_fwd(PrrtSocket *s);
-    uint32_t PrrtSocket_get_btlbw_back(PrrtSocket *s);
-
-    bint PrrtSocket_get_app_limited(PrrtSocket *socket)
+    uint32_t PrrtSocket_get_btldatarate_fwd(PrrtSocket *s);
+    uint32_t PrrtSocket_get_btldatarate_back(PrrtSocket *s);
+
+    uint32_t PrrtSocket_get_bbr_state(PrrtSocket *s)
+    uint64_t PrrtSocket_get_full_datarate(PrrtSocket *s)
+    bint PrrtSocket_get_filled_pipe(PrrtSocket *s)
+    uint32_t PrrtSocket_get_cycle_index(PrrtSocket *s)
+    float PrrtSocket_get_pacing_gain(PrrtSocket *s)
+    uint32_t PrrtSocket_get_cwnd(PrrtSocket *s)
+    uint32_t PrrtSocket_get_inflight(PrrtSocket *s)
+    uint32_t PrrtSocket_get_pacing_rate(PrrtSocket *s)
+    uint32_t PrrtSocket_get_send_quantum(PrrtSocket *s)
+    uint32_t PrrtSocket_get_pipe(PrrtSocket *s)
+    uint32_t PrrtSocket_get_delivered(PrrtSocket *s)
+    bint PrrtSocket_get_bbr_round_start(PrrtSocket *s)
+    uint32_t PrrtSocket_get_bbr_app_limited(PrrtSocket *socket)
+    bint PrrtSocket_get_bbr_is_app_limited(PrrtSocket *socket)
     bint PrrtSocket_enable_thread_pinning(PrrtSocket *socket)
 
 cdef extern from "proto/stores/packetDeliveryStore.h":
@@ -173,4 +188,3 @@ cdef extern from "util/pipe.h":
 cdef extern from "util/mpsc_queue.h":
     ctypedef struct MPSCQueue:
         pass
-

prrt/defines.h

@@ -24,14 +24,14 @@
 
 #define MAX_PAYLOAD_LENGTH 65507 // maximum UDP packet length (2^16 - 1 - 8 (UDP Header) - 20 (IP Header))
 
-#define SEQNO_SPACE UINT16_MAX // 2**16 as seqno is uint16_t
+#define SEQNO_SPACE (((uint32_t) UINT16_MAX)+1) // 2**16 as seqno is uint16_t
 
-#define TIMESTAMP_SPACE UINT32_MAX
+#define TIMESTAMP_SPACE (((uint64_t) UINT32_MAX)+1)
 
 #define GF_BITS 8
 
-#define K_START 4
-#define N_START 7
+#define K_START 1
+#define N_START 1
 #define N_P_START 1
 
 #define RRT_ALPHA 0.125

prrt/proto/CMakeLists.txt

 set (PRRT_SOURCES ../defines.h
-        types/block.c types/block.h
-        types/channelStateInformation.c types/channelStateInformation.h
+        bbr.c bbr.h
         clock.c clock.h
-        types/codingParams.c types/codingParams.h
         receiver.c receiver.h
         socket.c socket.h
-        types/applicationConstraints.c types/applicationConstraints.h
+        timer.c timer.h
         processes/dataReceiver.c processes/dataReceiver.h
         processes/dataTransmitter.c processes/dataTransmitter.h
         stores/dataPacketStore.c stores/dataPacketStore.h
         stores/deliveredPacketTable.c stores/deliveredPacketTable.h
         stores/inFlightPacketStore.c stores/inFlightPacketStore.h
+        stores/pace.c stores/pace.h
+        stores/paceFilter.c stores/paceFilter.h
         stores/packetTimeoutTable.c stores/packetTimeoutTable.h
         stores/packetDeliveryStore.c stores/packetDeliveryStore.h
         stores/receptionTable.c stores/receptionTable.h
         stores/repairBlockStore.c stores/repairBlockStore.h
-        types/packetTimeout.c types/packetTimeout.h
+        types/applicationConstraints.c types/applicationConstraints.h
+        types/block.c types/block.h
+        types/codingParams.c types/codingParams.h
+        types/channelStateInformation.c types/channelStateInformation.h
         types/lossStatistics.c types/lossStatistics.h
         types/packet.c types/packet.h
-        vdmcode/block_code.c vdmcode/block_code.h)
+        types/packetTimeout.c types/packetTimeout.h
+        vdmcode/block_code.c vdmcode/block_code.h types/packetTracking.c types/packetTracking.h types/rateSample.c types/rateSample.h)
 
 if (XLAP)
     set(PRRT_SOURCES ${PRRT_SOURCES} ../xlap/xlap.c ../xlap/xlap.h)
 endif()
 
-add_library(PRRT ${PRRT_SOURCES})
+add_library(prrt_obj OBJECT ${PRRT_SOURCES})
+set_property(TARGET prrt_obj PROPERTY POSITION_INDEPENDENT_CODE 1)
+
+
+add_library(prrt_static STATIC $<TARGET_OBJECTS:prrt_obj>)
+target_link_libraries(prrt_static rt)
+set_target_properties(prrt_static PROPERTIES OUTPUT_NAME "prrt")
+
+add_library(prrt_shared SHARED $<TARGET_OBJECTS:prrt_obj>)
+target_link_libraries(prrt_shared rt)
+set_target_properties(prrt_shared PROPERTIES OUTPUT_NAME "prrt")
 
-target_link_libraries(PRRT rt)
+install(TARGETS prrt_static prrt_shared
+        RUNTIME DESTINATION bin
+        LIBRARY DESTINATION lib
+        ARCHIVE DESTINATION lib)

prrt/proto/bbr.h

+#ifndef PRRT_BBR_H
+#define PRRT_BBR_H
+
+#include <stdint.h>
+#include "stdbool.h"
+#include "types/packet.h"
+#include "clock.h"
+#include "types/channelStateInformation.h"
+#include "types/packetTracking.h"
+#include "types/rateSample.h"
+#include "../util/windowedFilter.h"
+
+#define PROBE_GAIN 1.1
+#define DRAIN_GAIN 0.9
+#define RTpropFilterLen 2000000    // 2s
+#define BBRHighGain ((((float)2885) / 1000) + 1)
+#define BBRGainCycleLen 8
+#define ProbeRTTDuration 200000 //200ms
+#define RTprop_Inf UINT32_MAX
+
+enum bbr_state {
+    STARTUP,
+    DRAIN,
+    PROBE_DR,
+    PROBE_RTT
+};
+
+typedef struct bbr {
+    pthread_mutex_t lock;
+
+    prrtByteCount_t mps;
+    prrtByteCount_t min_pipe_cwnd;
+    prrtByteCount_t initial_cwnd;
+    prrtTimedelta_t rtprop;
+    prrtTimestamp_t rtprop_stamp;
+    prrtTimestamp_t probe_rtt_done_stamp;
+    bool probe_rtt_round_done;
+    bool packet_conservation;
+    prrtByteCount_t prior_cwnd;
+    prrtByteCount_t cwnd;
+    prrtByteCount_t target_cwnd;
+    bool idle_restart;
+
+    enum bbr_state state;
+
+    double pacing_gain;
+    float cwnd_gain;
+
+    bool filled_pipe;
+    prrtByteCount_t full_datarate;
+    uint32_t full_datarate_count;
+
+    double pacing_rate;
+    bool has_seen_rtt;
+
+    uint32_t next_round_delivered;
+    bool round_start;
+    uint32_t round_count;
+
+    uint32_t next_rtt_delivered;
+    uint32_t rtt_count;
+    bool rtprop_expired;
+
+    bool is_loss_recovery;
+    prrtTimestamp_t loss_recovery_stamp;
+
+    prrtTimestamp_t cycle_stamp;
+    uint8_t cycle_index;
+    float* pacing_gain_cycle;
+
+    prrtByteCount_t send_quantum;
+
+    prrtDeliveryRate_t datarate;
+
+    WindowedFilter* btlDatarateFilter;
+} BBR;
+
+BBR* BBR_Init(prrtByteCount_t maximum_payload_size);
+void BBR_OnACK(BBR* bbr, PrrtChannelStateInformation* csi, PrrtRateSample* rs, PrrtPacketTracking* packetTracking, prrtTimedelta_t rtt);
+void BBR_OnSpuriousLoss(BBR *bbr, PrrtPacketTracking *tracking);
+void BBR_OnRTOLoss(BBR *bbr);
+void BBR_OnLossExit(BBR *bbr);
+void BBR_destroy(BBR* bbr);
+
+double BBR_getPacingRate(BBR* bbr);
+prrtByteCount_t BBR_getCwnd(BBR* bbr);
+prrtDeliveryRate_t BBR_getBtlDatarate(BBR* bbr);
+uint32_t BBR_getState(BBR* bbr);
+prrtByteCount_t BBR_getFullDatarate(BBR* bbr);
+double BBR_getPacingGain(BBR* bbr);
+uint32_t BBR_getCycleIndex(BBR* bbr);
+bool BBR_getFilledPipe(BBR* bbr);
+uint32_t BBR_getRTProp(BBR* bbr);
+prrtByteCount_t BBR_getInflight(BBR* bbr);
+prrtByteCount_t BBR_getSendQuantum(BBR* bbr);
+bool BBR_getRoundStart(BBR* bbr);
+
+#endif //PRRT_BBR_H

prrt/proto/clock.c

 #include <sys/time.h>
 #include <stdlib.h>
+#include "../util/time.h"
 #include "../util/common.h"
 #include "clock.h"
 
@@ -19,50 +20,45 @@ prrtTimestamp_t PrrtClock_get_prrt_time_us(PrrtClock *clock)
         prrtTimestamp_t lastMeasurement = clock->lastMeasurement;
         prrtTimeDifference_t diff = diff_abs_ts(currentTime, lastMeasurement);
         prrtTimeDifference_t skew = (diff * clock->skew) / 400;
-        return (prrtTimestamp_t) virtualTime + clock->meanDeviation + skew;
+        return (prrtTimestamp_t) (virtualTime + clock->meanDeviation + skew);
     } else {
         return currentTime;
     }
 }
 
-
-bool PrrtClock_update(PrrtClock *clock, uint32_t referenceTime, uint32_t rtt)
+bool PrrtClock_update(PrrtClock *clock, uint32_t referenceTime, uint32_t propagation_delay)
 {
+    prrtTimestamp_t virtualTime = PrrtClock_get_prrt_time_us(clock);
     prrtTimestamp_t currentTime = PrrtClock_get_current_time_us();
-    prrtTimestamp_t virtualTime = clock->virtualTime;
-    prrtTimeDifference_t clockSkew = clock->skew;
-    prrtTimedelta_t delay = rtt/2; // half the rtt
-    prrtTimeDifference_t phaseError = referenceTime - virtualTime + delay;
-    //printf("V: %13u, C: %13u, S: %13d, R: %13u, E: %13d, D: %3u L: %u\n", virtualTime, currentTime, clockSkew, referenceTime, phaseError, delay, clock->lastMeasurement);
+    int64_t clockSkew = clock->skew;
+    prrtTimedelta_t delay = propagation_delay / 2; // half the rtt
+    int64_t phaseError = PrrtTimestamp_cmp(referenceTime, virtualTime) + delay;
+    //printf("V: %13u, C: %13u, S: %13d, R: %13u, E: %13ld, D: %3u L: %u\n", virtualTime, currentTime, clockSkew, referenceTime, phaseError, delay, clock->lastMeasurement);
 
-    if(abs(phaseError) > 10000) {
+    if(labs(phaseError) > 10000) {
         clock->meanDeviation = 0;
         clock->skew = 0;
         clock->lastMeasurement = currentTime;
         clock->virtualTime = currentTime;
     }
 
-    prrtTimeDifference_t deviationSum = (prrtTimeDifference_t) (phaseError + 3 * clock->meanDeviation); // TODO: why 3?
+    int64_t deviationSum =(phaseError + 3 * clock->meanDeviation); // TODO: why 3?
     prrtTimeDifference_t meanDeviation = (prrtTimeDifference_t) (deviationSum / 4); // TODO: find out why???
-    prrtTimeDifference_t period = diff_abs_ts(currentTime, clock->lastMeasurement);
+    int64_t period = PrrtTimestamp_cmp(currentTime, clock->lastMeasurement);
 
     if(period > 0) {
         clockSkew = (meanDeviation / period) + 15 * clock->skew;
         clockSkew = (prrtTimeDifference_t) (clockSkew / 16);
     }
 
-    virtualTime = virtualTime + meanDeviation + period * clockSkew / 400;
+    virtualTime = clock->virtualTime + meanDeviation + period * clockSkew / 400;
 
     clock->meanDeviation = meanDeviation;
-    clock->skew = clockSkew;
+    clock->skew = (prrtTimeDifference_t) clockSkew;
     clock->lastMeasurement = currentTime;
     clock->virtualTime = virtualTime;
 
-    if(abs(phaseError) > 10000) {
-        clock->virtualTime = currentTime;
-    }
-
-    //printf("V: %13u, C: %13u; S: %13d; M: %13d, P:%13u\n", virtualTime, currentTime, clockSkew, meanDeviation, period);
+    //printf("V: %13u, C: %13u; S: %13d; M: %13d, P:%13u\n\n", virtualTime, currentTime, clockSkew, meanDeviation, period);
     return true;
 }
 

prrt/proto/clock.h

@@ -26,7 +26,7 @@ prrtTimestamp_t PrrtClock_get_current_time_us(void);
 
 prrtTimestamp_t PrrtClock_get_prrt_time_us(PrrtClock *clock);
 
-bool PrrtClock_update(PrrtClock *clock, prrtTimestamp_t referenceTime, prrtTimedelta_t rtt);
+bool PrrtClock_update(PrrtClock *clock, prrtTimestamp_t referenceTime, prrtTimedelta_t propagation_delay);
 
 #define diff_abs_ts(timeA, timeB) (prrtTimeDifference_t) abs(((prrtTimeDifference_t) timeA) - ((prrtTimeDifference_t)timeB))
 

prrt/proto/processes/dataTransmitter.h

 #ifndef PRRT_DATA_TRANSMITTER_H
 #define PRRT_DATA_TRANSMITTER_H
 
-void * send_data_loop(void *ptr);
+#include "../socket.h"
+
+void * PrrtDataTransmitter_send_data_loop(void *ptr);
+void PrrtDataTransmitter_transmit(PrrtSocket *sock_ptr, PrrtPacket *packet);
 
 #endif //PRRT_DATA_TRANSMITTER_H

prrt/proto/receiver.h

@@ -7,28 +7,10 @@
 #include <sys/socket.h>
 #include <netdb.h>
 #include "stores/inFlightPacketStore.h"
+#include "bbr.h"
+#include "types/applicationConstraints.h"
 #include "types/channelStateInformation.h"
 
-typedef struct prrtRateSample {
-    prrtByteCount_t prior_delivered;
-    prrtTimestamp_t prior_time;
-    prrtTimedelta_t send_elapsed;
-    prrtTimedelta_t ack_elapsed;
-
-    prrtTimedelta_t interval;
-    prrtByteCount_t delivered;
-    bool is_app_limited;
-
-    prrtDeliveryRate_t delivery_rate; // Bps
-} PrrtRateSample;
-
-typedef struct packetTracking {
-    prrtByteCount_t pipe;
-    prrtByteCount_t delivered;
-    prrtTimestamp_t delivered_time;
-    prrtTimestamp_t first_sent_time;
-    prrtByteCount_t app_limited;
-} PrrtPacketTracking;
 
 typedef struct prrtReceiver {
     const char *host_name;
@@ -36,23 +18,39 @@ typedef struct prrtReceiver {
     struct addrinfo *ai;
     PrrtChannelStateInformation *csi;
     pthread_mutex_t lock;
+    pthread_cond_t wait_for_space;
+    atomic_bool closing;
+
+    BBR* bbr;
 
-    PrrtInFlightPacketStore *dataPacketStates;
-    PrrtInFlightPacketStore *redundancyPacketStates;
+    PrrtInFlightPacketStore *dataInflightPacketStore;
+    PrrtInFlightPacketStore *redundancyInflightPacketStore;
 
     PrrtRateSample *rateSample;
     PrrtPacketTracking *packetTracking;
 } PrrtReceiver;
 
-PrrtReceiver *PrrtReceiver_create(const char *host, uint16_t port);
-
-bool PrrtReceiver_updateAndGenerateRateSample(PrrtReceiver *recv, prrtSequenceNumber_t seqnum, uint8_t packetType,
-                                              prrtTimestamp_t receiveTime);
+PrrtReceiver *PrrtReceiver_create(const char *host, uint16_t port, prrtByteCount_t maximum_payload_size);
 
 void PrrtReceiver_add_outstanding_packet_state(PrrtReceiver *recv, PrrtPacket *packet, prrtTimestamp_t sentTime);
+prrtByteCount_t PrrtReceiver_get_pipe(PrrtReceiver *recv);
+void PrrtReceiver_rto_check(PrrtReceiver *recv, PrrtApplicationConstraints *constraints);
+
+void PrrtReceiver_on_application_write(PrrtReceiver *receiver, uint32_t send_queue_length, prrtSequenceNumber_t sequenceNumber);
+void PrrtReceiver_on_ack(PrrtReceiver *receiver, PrrtPacketFeedbackPayload *feedbackPayload, prrtTimestamp_t receiveTime,
+                         prrtTimedelta_t rtt, PrrtApplicationConstraints *constraints);
+
+bool PrrtReceiver_wait_for_space(PrrtReceiver *receiver, prrtByteCount_t maximum_payload_size,
+                                 PrrtApplicationConstraints *pConstraints);
+uint32_t PrrtReceiver_get_retransmission_delay(PrrtReceiver *socket, PrrtApplicationConstraints *applicationConstraints);
 
-void PrrtReceiver_on_application_write(PrrtReceiver* receiver);
+prrtByteCount_t PrrtReceiver_get_space(PrrtReceiver *receiver);
+double PrrtReceiver_get_BBR_pacingRate(PrrtReceiver *receiver);
+double PrrtReceiver_get_BBR_pacingGain(PrrtReceiver *receiver);
+uint32_t PrrtReceiver_get_BBR_state(PrrtReceiver *receiver);
+prrtDeliveryRate_t PrrtReceiver_get_BBR_btlDr(PrrtReceiver *receiver);
 
+void PrrtReceiver_interrupt(PrrtReceiver *receiver);
 bool PrrtReceiver_destroy(PrrtReceiver *receiver);
 
 #endif //PRRT_RECEIVER_H

prrt/proto/socket.h

 #ifndef PRRT_SOCKET_H
 #define PRRT_SOCKET_H
 
+
 #include "../defines.h"
 #include "../util/list.h"
 #include "../util/pipe.h"
@@ -9,6 +10,8 @@
 #include "stores/dataPacketStore.h"
 #include "stores/deliveredPacketTable.h"
 #include "stores/packetTimeoutTable.h"
+#include "stores/pace.h"
+#include "stores/paceFilter.h"
 #include "stores/receptionTable.h"
 #include "stores/repairBlockStore.h"
 #include "stores/packetDeliveryStore.h"
@@ -17,6 +20,7 @@
 #include "types/lossStatistics.h"
 #include "types/packet.h"
 #include "clock.h"
+#include "timer.h"
 #include "xlap.h"
 #include "receiver.h"
 
@@ -31,8 +35,12 @@ typedef struct prrtSocket {
 
     struct sockaddr_in *address;
     bool isBound;
+    bool withTimestamp;
+
+    bool pacingEnabled;
 
     PrrtClock clock;
+    PrrtBlock* receiveBlock;
 
     pthread_t sendDataThread;
     Pipe *sendDataQueue;
@@ -50,6 +58,8 @@ typedef struct prrtSocket {
 
     atomic_bool closing;
 
+    prrtTimestamp_t nextSendTime;
+
     prrtSequenceNumber_t packetsCount;
     prrtSequenceNumber_t sequenceNumberSource;
     prrtSequenceNumber_t sequenceNumberRepetition;
@@ -69,6 +79,15 @@ typedef struct prrtSocket {
     PrrtCodingConfiguration *codingParameters;
     PrrtCoder *coder;
 
+    // Pacing
+    PrrtPace* appSendPace;
+    PrrtPace* prrtTransmitPace;
+    PrrtPace* prrtReceivePace;
+    PrrtPace* appDeliverPace;
+
+    prrtTimedelta_t send_peer_btl_pace;
+    prrtTimedelta_t recv_peer_btl_pace;
+
     _Atomic (XlapTimestampTable *) tstable[2];
 
     pthread_attr_t *sendDataThreadAttr;
@@ -77,8 +96,12 @@ typedef struct prrtSocket {
     atomic_bool isHardwareTimestamping;
     char *interfaceName;
 
+    PrrtChannelStateInformation* senderChannelStateInformation;
+
     atomic_bool isThreadPinning;
     prrtByteCount_t maximum_payload_size;
+
+    PrrtTimer *retransmissionTimer;
 } PrrtSocket;
 
 
@@ -88,7 +111,7 @@ bool PrrtSocket_enable_hardware_timestamping(PrrtSocket *s, const char * interfa
 
 bool PrrtSocket_enable_thread_pinning(PrrtSocket *s);
 
-bool PrrtSocket_bind(PrrtSocket *s, const char *ipAddress, const uint16_t port);
+int PrrtSocket_bind(PrrtSocket *s, const char *ipAddress, const uint16_t port);
 
 bool PrrtSocket_set_sock_opt(PrrtSocket *s, const char *name, const uint32_t value);
 
@@ -106,7 +129,11 @@ int PrrtSocket_close(PrrtSocket *s);
 
 bool PrrtSocket_connect(PrrtSocket *s, const char *host, const uint16_t port);
 
-int PrrtSocket_send(PrrtSocket *s, const uint8_t *data, size_t data_len);
+int PrrtSocket_send_async(PrrtSocket *s, const uint8_t *data, const size_t data_len);
+int PrrtSocket_send_sync(PrrtSocket *s, const uint8_t *data, size_t data_len);
+int PrrtSocket_try_send_async(PrrtSocket *s, const uint8_t *data, const size_t data_len);
+
+bool PrrtSocket_pace(PrrtSocket *sock_ptr, bool prepace);
 
 int32_t PrrtSocket_recv(PrrtSocket *s, void *buf_ptr, struct sockaddr* addr);
 
@@ -118,7 +145,6 @@ int32_t PrrtSocket_receive_ordered(PrrtSocket *s, void *buf_ptr, struct sockaddr
 int32_t PrrtSocket_receive_ordered_wait(PrrtSocket *s, void *buf_ptr, struct sockaddr* addr, prrtTimedelta_t time_window_us);
 int32_t PrrtSocket_receive_ordered_timedwait(PrrtSocket *s, void *buf_ptr, struct sockaddr* addr, prrtTimedelta_t time_window_us, struct timespec* deadline);
 
-
 bool PrrtSocket_cleanup(PrrtSocket *s);
 
 bool PrrtSocket_closing(PrrtSocket *s);
@@ -126,11 +152,23 @@ bool PrrtSocket_closing(PrrtSocket *s);
 bool PrrtSocket_uses_thread_pinning(PrrtSocket *s);
 
 uint32_t PrrtSocket_get_rtprop_fwd(PrrtSocket *s);
-
 prrtPacketLossRate_t PrrtSocket_get_plr_fwd(PrrtSocket *s);
 prrtDeliveryRate_t PrrtSocket_get_delivery_rate_fwd(PrrtSocket *s);
-prrtDeliveryRate_t PrrtSocket_get_btlbw_fwd(PrrtSocket *s);
-prrtDeliveryRate_t PrrtSocket_get_btlbw_back(PrrtSocket *s);
-bool PrrtSocket_get_app_limited(PrrtSocket *s);
+prrtDeliveryRate_t PrrtSocket_get_btldatarate_fwd(PrrtSocket *s);
+prrtDeliveryRate_t PrrtSocket_get_btldatarate_back(PrrtSocket *s);
+uint64_t PrrtSocket_get_full_datarate(PrrtSocket *s);
+bool PrrtSocket_get_filled_pipe(PrrtSocket *s);
+uint32_t PrrtSocket_get_bbr_state(PrrtSocket *s);
+uint32_t PrrtSocket_get_bbr_app_limited(PrrtSocket *s);
+bool PrrtSocket_get_bbr_is_app_limited(PrrtSocket *s);
+uint32_t PrrtSocket_get_cycle_index(PrrtSocket *s);
+float PrrtSocket_get_pacing_gain(PrrtSocket *s);
+uint32_t PrrtSocket_get_cwnd(PrrtSocket *s);
+uint32_t PrrtSocket_get_inflight(PrrtSocket *s);
+uint32_t PrrtSocket_get_pacing_rate(PrrtSocket *s);
+prrtByteCount_t PrrtSocket_get_send_quantum(PrrtSocket *s);
+prrtByteCount_t PrrtSocket_get_pipe(PrrtSocket *s);
+prrtByteCount_t PrrtSocket_get_delivered(