Adding congestion control and ARQ.

.gitlab-ci.yml

@@ -38,6 +38,17 @@ build:container:
         - docker push $CI_REGISTRY_IMAGE:$DOCKER_TAG
         - docker rmi $CI_REGISTRY_IMAGE:$DOCKER_TAG
 
+build:container_tcp:
+    stage: build
+    tags:
+        - docker
+    script:
+        - export DOCKER_TAG=$(echo "$CI_BUILD_REF_NAME""_tcp" | sed 's#/#_#' | sed 's#^master$#latest#')
+        - docker build -t $CI_REGISTRY_IMAGE:$DOCKER_TAG --build-arg http_proxy=http://www-proxy.uni-saarland.de:3128 -f docker/Dockerfile_tcp .
+        - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+        - docker push $CI_REGISTRY_IMAGE:$DOCKER_TAG
+        - docker rmi $CI_REGISTRY_IMAGE:$DOCKER_TAG
+
 test:prrt_mem:
   stage: test
   dependencies:

README.md

@@ -5,6 +5,7 @@
 ## Features
 
 * Hybrid error control (FEC + ARQ) using systematic Vandermonde codes
+* Congestion control using a variant of [BBR](https://groups.google.com/forum/#!forum/bbr-dev)
 * Clock synchronization between sending stack and receiving stack
 * Applications can specify packet-level expiration times
 * Different receive modes for ASAP and time-synchronized operation
@@ -29,9 +30,9 @@ port = int(sys.argv[1])
 s = prrt.PrrtSocket(port=port)
 
 while True:
-    d = s.recv()
+    d = s.recv().decode("utf8")
     if d != "Close":
-        print d
+        print(d)
     else:
         break
 ```
@@ -44,25 +45,27 @@ import prrt
 
 host = sys.argv[1]
 port = int(sys.argv[2])
+localport = int(sys.argv[3])
 
-s = prrt.PrrtSocket(port=port)
+s = prrt.PrrtSocket(port=localport)
 s.connect(host, port)
 
+
 for i in range(10):
-    s.send("Packet {}".format(i))
-s.send("Close")
+    s.send("Packet {}".format(i).encode("utf8"))
+s.send("Close".encode("utf8"))
 ```
 
 Start the receiver by:
 
 ```bash
-python receiver.py 5000
+python3 receiver.py 5000
 ```
 
 In a separate terminal, run:
 
 ```bash
-python sender.py 127.0.0.1 5000
+python3 sender.py 127.0.0.1 5000 6000
 ```
 
 This should generate the following output in the receiver console:
@@ -84,13 +87,13 @@ Packet 9
 If you find PRRT useful and incorporate it in your works, we are very happy to hear about it. Please also consider to cite us like this:
 
 ```bibtex
-@misc{sic2018prrt, 
-      author = {Schmidt, Andreas}, 
-      title = {PRRT: Predictably Reliable Real-time Transport}, 
-      howpublished={Web page}, 
-      url = {http://prrt.larn.systems}, 
-      year = {2018} 
-} 
+@misc{sic2018prrt,
+      author = {Schmidt, Andreas},
+      title = {PRRT: Predictably Reliable Real-time Transport},
+      howpublished={Web page},
+      url = {http://prrt.larn.systems},
+      year = {2018}
+}
 ```
 
 ## License

dissect/prrt.lua

@@ -89,9 +89,16 @@ local ex_type = Field.new("prrt.type")
 local function getType() return ex_type()() end
 local function getTypeName() return prrtPacketTypeNames[getType()] end
 
+local ex_index = Field.new("prrt.index")
+local function getIndex() return ex_index()() end
+
 local ex_data_length = Field.new("prrt.data.length")
 local function getDataLength() return ex_data_length()() end
 
+local ex_red_baseseqno = Field.new("prrt.redundancy.baseSequenceNumber")
+local function getRedBaseSeqNo() return ex_red_baseseqno()() end
+
+
 local ex_red_n = Field.new("prrt.redundancy.n")
 local function getRedN() return ex_red_n()() end
 
@@ -109,7 +116,7 @@ local function dissect_data(buffer, pinfo, root)
     tree:add(pf_data_groupRTprop, buffer:range(8,4))
     tree:add(pf_data_packettimeout, buffer:range(12,4))
 
-    local label = "DATA Len=" .. getDataLength()
+    local label = "[D] Idx=" .. getIndex() .. " Len=" .. getDataLength()
     tree:set_text(label)
     pinfo.cols.info:set(label)
 end
@@ -121,7 +128,7 @@ local function dissect_redundancy(buffer, pinfo, root)
     tree:add(pf_red_n, buffer:range(6,1))
     tree:add(pf_red_k, buffer:range(7,1))
 
-    local label = "REDUNDANCY n=" .. getRedN() .. " k=" .. getRedK()
+    local label = "[R] Idx=" .. getIndex() .. " b=" .. getRedBaseSeqNo() .. " n=" .. getRedN() .. " k=" .. getRedK()
     tree:set_text(label)
     pinfo.cols.info:set(label)
 end
@@ -139,7 +146,7 @@ local function dissect_feedback(buffer, pinfo, root)
     tree:add(pf_fb_acktype, buffer:range(20,1))
     tree:add(pf_fb_ackSeqN, buffer:range(21, 2))
 
-    local label = "FEEDBACK"
+    local label = "[F]"
     tree:set_text(label)
     pinfo.cols.info:set(label)
 end

docker/Dockerfile

@@ -2,9 +2,13 @@ FROM gcc:5
 
 MAINTAINER Andreas Schmidt <schmidt@nt.uni-saarland.de>
 
-RUN apt-get update \
-    && apt-get upgrade -y \
-    && apt-get install -y cmake
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install --yes --force-yes \
+    bc \
+    cmake \
+    traceroute \
+    tshark
 
 COPY CMakeLists.txt /prrt/
 COPY prrt /prrt/prrt
@@ -15,8 +19,7 @@ WORKDIR /prrt
 
 RUN cmake . \
     && make
-ENV PATH /prrt/bin:$PATH
-WORKDIR /prrt/bin
+ENV PATH /prrt:$PATH
 
 VOLUME /output
 

docker/Dockerfile_tcp

+FROM gcc:5
+
+MAINTAINER Andreas Schmidt <schmidt@nt.uni-saarland.de>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install --yes --force-yes \
+    bc \
+    cmake \
+    traceroute \
+    tshark
+
+COPY CMakeLists.txt /prrt/
+COPY prrt /prrt/prrt
+COPY tests /prrt/tests
+COPY docker/entrypoint.sh /
+
+WORKDIR /prrt
+
+RUN cmake -DTCP=1 . \
+    && make
+ENV PATH /prrt:$PATH
+
+VOLUME /output
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/entrypoint.sh

@@ -5,13 +5,15 @@ dev=eth0
 command=$1
 shift
 
-if [[ "$command" == "sender" || "$command" == "receiver" ]]; then
+if [[ "$command" == "sender" || "$command" == "receiver" || "$command" == "time-sender" || "$command" == "time-receiver" ]]; then
     :
 else
     echo "Command should be either sender or receiver."
     exit 0;
 fi
 
+TARGET="127.0.0.1"
+OUTPUT="/dev/null"
 NETEM=()
 PRRT=()
 while [[ $# -gt 0 ]]
@@ -19,7 +21,24 @@ do
 key="$1"
 
 case $key in
-    -t|--target|-p|--port|-r|--rounds)
+    -t|--target)
+    if [[  "$command" == "sender" || "$command" == "time-sender" ]]; then
+    PRRT+=("$1 $2")
+    fi
+    TARGET=("$2")
+    shift
+    shift
+    ;;
+    -w|--wireshark)
+    OUTPUT=("$2")
+    shift
+    shift
+    ;;
+    -T|--threadpinning)
+    PRRT+=("$1")
+    shift
+    ;;
+    -p|--port|-r|--rounds|-s|--size|-R|--rcvbuf|-S|--sndbuf|-o|--output|-a|--appdelay)
     PRRT+=("$1 $2")
     shift
     shift
@@ -34,8 +53,31 @@ done
 PRRT_PARAMS="${PRRT[@]}"
 NETEM_PARAMS="${NETEM[@]}"
 
+echo "Starting Wireshark."
+tshark -i eth0 -w $OUTPUT.pcap &
+TSHARK_PID=$!
+sleep 5
+
+start=$(date +%s.%N);
+echo "Checking reachability of $TARGET."
+until ping -c1 $TARGET &>/dev/null; sleep 1; do :; done
+dur=$(echo "$(date +%s.%N) - $start" | bc);
+printf "Reachable after %.6f seconds\n" $dur
+
+traceroute $TARGET > $OUTPUT.tr
+echo "Traceroute done."
+
+if [[ "$command" == "sender" || "$command" == "time-sender" ]]; then
+    echo "Delaying sender start."
+else
+    echo "Delaying receiver start."
+fi
+
 echo "Running PRRT with command: \"$command $PRRT_PARAMS\" and link parameters: \"$NETEM_PARAMS\""
 
 tc qdisc add dev $dev root netem $NETEM_PARAMS
-/prrt/$command $PRRT_PARAMS -o /output/log.csv
+/prrt/$command $PRRT_PARAMS
+echo "Done."
 tc qdisc del dev $dev root
+
+kill $TSHARK_PID

prrt/CMakeLists.txt

@@ -10,14 +10,26 @@ if (XLAP)
     add_definitions(-DXLAP)
 endif()
 
+option (TCP "Set time protocol to TCP.")
+
+if (TCP)
+    add_definitions(-DTCP)
+endif()
+
 
 add_subdirectory(proto)
 add_subdirectory(util)
 
 add_executable(sender sender.c)
 add_executable(receiver receiver.c)
-add_executable(refcount refcount.c)
+add_executable(time-sender time-sender.c)
+add_executable(time-receiver time-receiver.c)
 
 target_link_libraries(sender LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})
 target_link_libraries(receiver LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})
+
+target_link_libraries(time-sender LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(time-receiver LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})
+
+add_executable(refcount refcount.c)
 target_link_libraries(refcount LINK_PUBLIC PRRT UTIL ${CMAKE_THREAD_LIBS_INIT})

prrt/cprrt.pxd

@@ -119,11 +119,13 @@ cdef extern from "proto/socket.h":
 
     ctypedef prrtSocket PrrtSocket
 
-    cdef PrrtSocket* PrrtSocket_create(const uint32_t mtu, const uint32_t target_delay)
+    cdef PrrtSocket* PrrtSocket_create(const uint32_t maximum_payload_size, const uint32_t target_delay)
     bint PrrtSocket_bind(PrrtSocket *sock_ptr, const_char *ipAddress, const uint16_t port)
     int PrrtSocket_close(const PrrtSocket *sock_ptr)
     int PrrtSocket_connect(PrrtSocket *sock_ptr, const_char *host, const uint16_t port)
-    int PrrtSocket_send(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len)
+    int PrrtSocket_send_sync(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len)
+    int PrrtSocket_send_async(PrrtSocket *sock_ptr, const uint8_t *data, const size_t data_len)
+
 
     int32_t PrrtSocket_recv(PrrtSocket *sock_ptr, void *buf_ptr, sockaddr* addr) nogil
     int32_t PrrtSocket_receive_asap(PrrtSocket *s, void *buf_ptr, sockaddr* addr) nogil
@@ -144,11 +146,23 @@ cdef extern from "proto/socket.h":
     uint32_t PrrtSocket_get_rtprop_fwd(PrrtSocket *socket)
     float PrrtSocket_get_plr_fwd(PrrtSocket *socket)
     uint32_t PrrtSocket_get_delivery_rate_fwd(PrrtSocket *socket)
-
     uint32_t PrrtSocket_get_btlbw_fwd(PrrtSocket *s);
     uint32_t PrrtSocket_get_btlbw_back(PrrtSocket *s);
 
-    bint PrrtSocket_get_app_limited(PrrtSocket *socket)
+    uint32_t PrrtSocket_get_bbr_state(PrrtSocket *s)
+    uint64_t PrrtSocket_get_full_bw(PrrtSocket *s)
+    bint PrrtSocket_get_filled_pipe(PrrtSocket *s)
+    uint32_t PrrtSocket_get_cycle_index(PrrtSocket *s)
+    float PrrtSocket_get_pacing_gain(PrrtSocket *s)
+    uint32_t PrrtSocket_get_cwnd(PrrtSocket *s)
+    uint32_t PrrtSocket_get_inflight(PrrtSocket *s)
+    uint32_t PrrtSocket_get_pacing_rate(PrrtSocket *s)
+    uint32_t PrrtSocket_get_send_quantum(PrrtSocket *s)
+    uint32_t PrrtSocket_get_pipe(PrrtSocket *s)
+    uint32_t PrrtSocket_get_delivered(PrrtSocket *s)
+    bint PrrtSocket_get_bbr_round_start(PrrtSocket *s)
+    uint32_t PrrtSocket_get_bbr_app_limited(PrrtSocket *socket)
+    bint PrrtSocket_get_bbr_is_app_limited(PrrtSocket *socket)
     bint PrrtSocket_enable_thread_pinning(PrrtSocket *socket)
 
 cdef extern from "proto/stores/packetDeliveryStore.h":

prrt/proto/CMakeLists.txt

 set (PRRT_SOURCES ../defines.h
-        types/block.c types/block.h
-        types/channelStateInformation.c types/channelStateInformation.h
+        bbr.c bbr.h
         clock.c clock.h
-        types/codingParams.c types/codingParams.h
         receiver.c receiver.h
         socket.c socket.h
-        types/applicationConstraints.c types/applicationConstraints.h
+        timer.c timer.h
         processes/dataReceiver.c processes/dataReceiver.h
         processes/dataTransmitter.c processes/dataTransmitter.h
         stores/dataPacketStore.c stores/dataPacketStore.h
@@ -15,10 +13,14 @@ set (PRRT_SOURCES ../defines.h
         stores/packetDeliveryStore.c stores/packetDeliveryStore.h
         stores/receptionTable.c stores/receptionTable.h
         stores/repairBlockStore.c stores/repairBlockStore.h
-        types/packetTimeout.c types/packetTimeout.h
+        types/applicationConstraints.c types/applicationConstraints.h
+        types/block.c types/block.h
+        types/codingParams.c types/codingParams.h
+        types/channelStateInformation.c types/channelStateInformation.h
         types/lossStatistics.c types/lossStatistics.h
         types/packet.c types/packet.h
-        vdmcode/block_code.c vdmcode/block_code.h)
+        types/packetTimeout.c types/packetTimeout.h
+        vdmcode/block_code.c vdmcode/block_code.h types/packetTracking.c types/packetTracking.h types/rateSample.c types/rateSample.h)
 
 if (XLAP)
     set(PRRT_SOURCES ${PRRT_SOURCES} ../xlap/xlap.c ../xlap/xlap.h)

prrt/proto/bbr.c

+#include "bbr.h"
+#include "../util/dbg.h"
+#include "../util/common.h"
+#include "receiver.h"
+#include <math.h>
+
+prrtByteCount_t BBR_Inflight(BBR* bbr, double gain)
+{
+    if (bbr->rtprop == RTprop_Inf)
+        return bbr->initial_cwnd; /* no valid RTT samples yet */
+    uint32_t quanta = bbr->mps;
+    uint32_t estimated_bdp = (uint32_t) round((((double)bbr->bw) * bbr->rtprop) / (1000 * 1000));
+    return (uint32_t)(gain * estimated_bdp + quanta);
+}
+
+void BBR_EnterStartup(BBR* bbr)
+{
+    bbr->state = STARTUP;
+    bbr->pacing_gain = BBRHighGain;
+    bbr->cwnd_gain = BBRHighGain;
+}
+
+void BBR_UpdateBtlBw(BBR* bbr, PrrtRateSample* rs, PrrtPacketTracking* tracking)
+{
+    if (tracking->delivered >= bbr->next_round_delivered) {
+        bbr->next_round_delivered = tracking->delivered;
+        bbr->round_count++;
+        bbr->round_start = true;
+    } else {
+        bbr->round_start = false;
+    }
+    uint32_t delivery_rate_Bps = (uint32_t)((float) rs->delivery_rate);
+    if ((delivery_rate_Bps >= bbr->bw || !rs->is_app_limited) && delivery_rate_Bps != 0) {
+        bbr->bw = (uint32_t)WindowedFilter_push(bbr->btlBwFilter, delivery_rate_Bps);
+        debug(DEBUG_BBR, "Current BtlBw: %u, RS delivery rate: %u", bbr->bw, delivery_rate_Bps);
+    }
+}
+
+void BBR_CheckFullPipe(BBR* bbr, PrrtRateSample* rs)
+{
+    if (bbr->filled_pipe || !bbr->round_start || rs->is_app_limited)
+        return;  // no need to check for a full pipe now
+    if (bbr->bw >= bbr->full_bw * PROBE_GAIN) {  // BBR.BtlBw still growing?
+        bbr->full_bw = bbr->bw;    // record new baseline level
+        bbr->full_bw_count = 0;
+        return;
+    }
+    bbr->full_bw_count++; // another round w/o much growth
+    if (bbr->full_bw_count >= 3)
+        bbr->filled_pipe = true;
+}
+
+bool BBR_IsNextCyclePhase(BBR* bbr, prrtByteCount_t bytes_lost, prrtByteCount_t prior_inflight)
+{
+    bool is_full_length = (PrrtClock_get_current_time_us() - bbr->cycle_stamp) > bbr->rtprop;
+    if (bbr->pacing_gain == 1)
+        return is_full_length;
+    if (bbr->pacing_gain > 1)
+        return is_full_length && (bytes_lost > 0 || prior_inflight >= BBR_Inflight(bbr, bbr->pacing_gain));
+    bool is_max_length = (PrrtClock_get_current_time_us() - bbr->cycle_stamp) > 4 * bbr->rtprop;
+    return is_max_length || (prior_inflight <= BBR_Inflight(bbr, 1.0));
+}
+
+void BBR_AdvanceCyclePhase(BBR* bbr)
+{
+    bbr->cycle_stamp = PrrtClock_get_current_time_us();
+    bbr->cycle_index = (uint8_t )((bbr->cycle_index + 1) % BBRGainCycleLen);
+    float pacing_gain_cycle[BBRGainCycleLen] = {PROBE_GAIN, DRAIN_GAIN, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0};
+    bbr->pacing_gain = pacing_gain_cycle[bbr->cycle_index];
+    debug(DEBUG_BBR, "Advanced cycle with gain: %f", bbr->pacing_gain);
+}
+
+void BBR_CheckCyclePhase(BBR* bbr, prrtByteCount_t bytes_lost, prrtByteCount_t prior_inflight) {
+    if (bbr->state == PROBE_BW && BBR_IsNextCyclePhase(bbr, bytes_lost, prior_inflight))
+        BBR_AdvanceCyclePhase(bbr);
+}
+
+void BBR_EnterProbeBW(BBR* bbr)
+{
+    bbr->state = PROBE_BW;
+    bbr->pacing_gain = 1;
+    bbr->cwnd_gain = 2;
+    bbr->cycle_index = (uint8_t)(BBRGainCycleLen - 1 - (random() % 7));
+    BBR_AdvanceCyclePhase(bbr);
+}
+
+void BBR_CheckDrain(BBR* bbr, prrtByteCount_t bytes_inflight)
+{
+    if (bbr->state == STARTUP && bbr->filled_pipe) {
+        //Drain
+        bbr->state = DRAIN;
+        bbr->pacing_gain = 1 / BBRHighGain;  // pace slowly
+        bbr->cwnd_gain = BBRHighGain;    // maintain cwnd
+    }
+    if (bbr->state == DRAIN && bytes_inflight <= BBR_Inflight(bbr, 1.0))
+        BBR_EnterProbeBW(bbr);  // we estimate queue is drained
+}
+
+void BBR_ExitProbeRTT(BBR* bbr)
+{
+    if (bbr->filled_pipe)
+        BBR_EnterProbeBW(bbr);
+    else
+        BBR_EnterStartup(bbr);
+}
+
+uint32_t BBR_SaveCwnd(BBR* bbr)
+{
+    if (!bbr->is_loss_recovery && bbr->state != PROBE_RTT)
+        return bbr->cwnd;
+    return MAX(bbr->prior_cwnd ,bbr->cwnd);
+}
+
+void BBR_RestoreCwnd(BBR* bbr)
+{
+    bbr->cwnd = MAX(bbr->cwnd, bbr->prior_cwnd);
+}
+
+void BBR_UpdateRTprop(BBR* bbr, prrtTimedelta_t rtt)
+{
+    bbr->rtprop_expired = PrrtClock_get_current_time_us() > (bbr->rtprop_stamp + RTpropFilterLen);
+    if (rtt >= 0 && (rtt <= bbr->rtprop || bbr->rtprop_expired)) {
+        bbr->rtprop = rtt;
+        bbr->rtprop_stamp = PrrtClock_get_current_time_us();
+    }
+}
+
+void BBR_EnterProbeRTT(BBR *bbr) {
+    bbr->state = PROBE_RTT;
+    bbr->pacing_gain = 1;
+    bbr->cwnd_gain = 1;
+}
+
+void BBR_HandleProbeRTT(BBR *bbr, PrrtPacketTracking *tracking) {
+    tracking->app_limited = (tracking->delivered + tracking->pipe) ? : 1;
+    /* Ignore low rate samples during ProbeRTT: */
+    prrtTimestamp_t now = PrrtClock_get_current_time_us();
+    if (bbr->probe_rtt_done_stamp == 0 && tracking->pipe <= bbr->min_pipe_cwnd) {
+        bbr->probe_rtt_done_stamp = now + ProbeRTTDuration;
+        bbr->probe_rtt_round_done = false;
+        bbr->next_round_delivered = tracking->delivered;
+    } else if (bbr->probe_rtt_done_stamp != 0) {
+        if (bbr->round_start) {
+            bbr->probe_rtt_round_done = true;
+        }
+        if (bbr->probe_rtt_round_done && (now > bbr->probe_rtt_done_stamp)) {
+            bbr->rtprop_stamp = now;
+            BBR_RestoreCwnd(bbr);
+            BBR_ExitProbeRTT(bbr);
+        }
+    }
+}
+
+void BBR_CheckProbeRTT(BBR *bbr, PrrtPacketTracking *tracking) {
+
+    if (bbr->state != PROBE_RTT && bbr->rtprop_expired && !bbr->idle_restart) {
+        BBR_EnterProbeRTT(bbr);
+        BBR_SaveCwnd(bbr);
+        bbr->probe_rtt_done_stamp = 0;
+    }
+
+    if (bbr->state == PROBE_RTT) {
+        BBR_HandleProbeRTT(bbr, tracking);
+    }
+    bbr->idle_restart = false;
+}
+
+void BBR_UpdateModelAndState(BBR *bbr, PrrtRateSample *rs, PrrtPacketTracking *packetTracking, prrtTimedelta_t rtt)
+{
+    BBR_UpdateBtlBw(bbr, rs, packetTracking);
+    BBR_CheckCyclePhase(bbr, packetTracking->bytes_lost, packetTracking->prior_inflight);
+    BBR_CheckFullPipe(bbr, rs);
+    BBR_CheckDrain(bbr, packetTracking->pipe);
+    BBR_UpdateRTprop(bbr, rtt);
+    BBR_CheckProbeRTT(bbr, packetTracking);
+}
+
+void BBR_UpdateTargetCwnd(BBR* bbr)
+{
+    bbr->target_cwnd = BBR_Inflight(bbr, bbr->cwnd_gain);
+}
+
+void BBR_ModulateCwndForProbeRTT(BBR* bbr)
+{
+    if (bbr->state == PROBE_RTT)
+        bbr->cwnd = MIN(bbr->cwnd, bbr->min_pipe_cwnd);
+}
+
+void BBR_ModulateCwndForRecovery(BBR* bbr, prrtByteCount_t bytes_lost, prrtByteCount_t pipe, prrtByteCount_t delivered)
+{
+    if (bytes_lost > 0) {
+        if (bbr->cwnd > bytes_lost) {
+            bbr->cwnd = MAX(bbr->cwnd - bytes_lost, bbr->mps);
+        } else {
+            bbr->cwnd = bbr->mps;
+        }
+    }
+
+    if (bbr->packet_conservation)
+        bbr->cwnd = MAX(bbr->cwnd, pipe + delivered);
+}
+
+void BBR_SetCwnd(BBR* bbr, PrrtPacketTracking* packetTracking)
+{
+    BBR_UpdateTargetCwnd(bbr);
+    BBR_ModulateCwndForRecovery(bbr, packetTracking->bytes_lost, packetTracking->pipe, packetTracking->delivered);
+    if (!bbr->packet_conservation) {
+        if (bbr->filled_pipe)
+            bbr->cwnd = MIN(bbr->cwnd + packetTracking->delivered, bbr->target_cwnd);
+        else if (bbr->cwnd < bbr->target_cwnd || packetTracking->delivered < bbr->initial_cwnd)
+            bbr->cwnd = bbr->cwnd + packetTracking->delivered;
+        bbr->cwnd = MAX(bbr->cwnd, bbr->min_pipe_cwnd);
+    }
+    BBR_ModulateCwndForProbeRTT(bbr);
+    debug(DEBUG_BBR, "New cwnd: %u, State: %u", bbr->cwnd, bbr->state);
+}
+
+void BBR_SetPacingRateWithGain(BBR* bbr, double pacing_gain)
+{
+    double rate = (pacing_gain * ((double)bbr->bw));
+    debug(DEBUG_BBR, "Current rate: %f, Pacing gain: %f, BtlBw: %u, Calc Rate: %f, Filled pipe: %u", bbr->pacing_rate,
+          pacing_gain, bbr->bw, rate, bbr->filled_pipe);
+    if (rate != 0 && (bbr->filled_pipe || rate > bbr->pacing_rate))
+        bbr->pacing_rate = rate;
+}
+
+void BBR_SetPacingRate(BBR* bbr)
+{
+    BBR_SetPacingRateWithGain(bbr, bbr->pacing_gain);
+}
+
+void BBR_SetSendQuantum(BBR* bbr) {
+    if(bbr->pacing_rate < 150000) { // 1.2Mbps = 0.15 MBps = 150000 Bps
+        bbr->send_quantum = 1 * bbr->mps;
+    } else if (bbr->pacing_rate < 3000000) { // 24 Mbps = 20 * 1.2Mbps = 3000000
+        bbr->send_quantum = 2 * bbr->mps;
+    } else {
+        bbr->send_quantum = MIN((prrtByteCount_t) round((double) bbr->pacing_rate / 1000), 64000);
+    }
+}
+
+void BBR_OnACK(BBR* bbr, PrrtChannelStateInformation* csi, PrrtRateSample* rs, PrrtPacketTracking* packetTracking, prrtTimedelta_t rtt)
+{
+    check(pthread_mutex_lock(&bbr->lock) == 0, "Lock failed.");
+    BBR_UpdateModelAndState(bbr, rs, packetTracking, rtt);
+
+    BBR_SetPacingRate(bbr);
+    BBR_SetSendQuantum(bbr);
+    BBR_SetCwnd(bbr, packetTracking);
+    check(pthread_mutex_unlock(&bbr->lock) == 0, "Unlock failed.");
+    return;
+    error:
+    PERROR("BBR_OnACK failed.")
+}
+
+void BBR_OnSpuriousLoss(BBR *bbr, PrrtPacketTracking *tracking) {
+    check(pthread_mutex_lock(&bbr->lock) == 0, "Lock failed.");
+    if(!bbr->is_loss_recovery) {
+        bbr->is_loss_recovery = true;
+        bbr->loss_recovery_stamp = PrrtClock_get_current_time_us();