Commit bfe7ff49 authored by Andreas Schmidt's avatar Andreas Schmidt
Browse files

Refactored. Add reference times as parameter.

parent a7cad9a3
...@@ -2,10 +2,20 @@ ...@@ -2,10 +2,20 @@
## Conventions ## Conventions
* Stampnames:
`_T`: Timestamp (us precision) `_T`: Timestamp (us precision)
`_C`: Clockstamp `_C`: Clockstamp
`_D`: Duration (us precision) `_D`: Duration (us precision)
* Reserved Names:
* `Channel`
* `Sender`
* `Receiver`
* `EndToEnd`
## How to use X-lap? ## How to use X-lap?
### Step 0: Define your relevant timestamps across a packet trace. ### Step 0: Define your relevant timestamps across a packet trace.
......
data_files: data_files:
sender: "rtn2017/results/on/2017_03_28_09_33_00_Sender.csv" sender: "rtn2017/results/on/2017_03_28_09_33_00_Sender.csv"
receiver: "rtn2017/results/on/2017_03_28_09_33_00_Receiver.csv" receiver: "rtn2017/results/on/2017_03_28_09_33_00_Receiver.csv"
cycle_reference:
sender:
Start: PrrtSendStart
Stop: LinkTransmitEnd
receiver:
Start: PrrtReceivePackage
Stop: PrrtDeliver
time_reference:
sender:
Start: PrrtSendStart
Stop: LinkTransmitEnd
receiver:
Start: LinkReceive
Stop: PrrtDeliver
stamps: stamps:
PrrtSendStart: PrrtSendStart:
Source: sender Source: sender
......
# TODO: Refactor. # TODO: Refactor.
def get_outlier_treshold(stats):
def get_outlier_threshold(stats):
q75 = stats["75%"] q75 = stats["75%"]
iqr = q75 - stats["25%"] iqr = q75 - stats["25%"]
return q75 + 1.5 * iqr return q75 + 1.5 * iqr
def _dn(x): def _dn(x):
return x + "Time" return x + "_D"
def _filter(x, durations, source): def _filter(x, durations, source):
return durations[x]["Source"] == source return durations[x]["Source"] == source
def extract_durations(config): def extract_durations(config):
durations = config["durations"] durations = config["durations"]
......
import pandas as pd import pandas as pd
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (16,9)
plt.rcParams["figure.figsize"] = (16, 9)
plt.rcParams.update({'figure.autolayout': True}) plt.rcParams.update({'figure.autolayout': True})
from xlap.analyse.common import get_outlier_treshold, extract_durations from xlap.analyse.common import get_outlier_threshold, extract_durations
from xlap.analyse import box from xlap.analyse import box
# TODO: Refactor. # TODO: Refactor.
def _dn(x): def _dn(x):
return x + "Time" return x + "Time"
def _filter(x, durations, source): def _filter(x, durations, source):
return durations[x]["Source"] == source return durations[x]["Source"] == source
def jitter_causes(df, export=False, file_name=None): def jitter_causes(df, export=False, file_name=None):
stats = df["EndToEndTime"].describe() stats = df["EndToEndTime"].describe()
tresh = get_outlier_treshold(stats) tresh = get_outlier_threshold(stats)
outliers = df[df["EndToEndTime"] > tresh] outliers = df[df["EndToEndTime"] > tresh]
# TODO: Refactor lines out. # TODO: Refactor lines out.
reasons = ["SendTime", reasons = ["Send_D",
"PrrtTransmitTime", "PrrtTransmit_D",
"LinkTransmitTime", "LinkTransmit_D",
"SubmitTime", "Submit_D",
"SenderIPCTime", "SenderIPC_D",
"SenderEnqueuedTime", "SenderEnqueued_D",
#"EncodingTime", # "EncodingTime",
"EnqueueTime", "Enqueue_D",
"DecodingTime", "Decoding_D",
"HandlePacketTime", "HandlePacket_D",
"ReceiverIPCTime", "ReceiverIPC_D",
"FeedbackTime"] "Feedback_D"]
df_reasons = pd.DataFrame(index=outliers.index) df_reasons = pd.DataFrame(index=outliers.index)
for r in reasons: for r in reasons:
r_tresh = get_outlier_treshold(df[r].describe()) r_tresh = get_outlier_threshold(df[r].describe())
df_reasons[r] = 0 df_reasons[r] = 0
df_reasons[r] = outliers[outliers[r] > r_tresh].notnull() df_reasons[r] = outliers[outliers[r] > r_tresh].notnull()
df_sum = df_reasons.sum().sort_values(ascending=False) df_sum = df_reasons.sum().sort_values(ascending=False)
ax = df_sum.plot.bar(x="Reason",y="Frequency",rot=45,grid=True,legend=False,color="black") ax = df_sum.plot.bar(x="Reason", y="Frequency", rot=45, grid=True, legend=False, color="black")
fig=ax.get_figure() fig = ax.get_figure()
plt.ylabel("Frequency") plt.ylabel("Frequency")
ax.set_xticklabels(list(map(lambda x: x.get_text().replace("_D", ""), ax.get_xticklabels()))) ax.set_xticklabels(list(map(lambda x: x.get_text().replace("_D", ""), ax.get_xticklabels())))
fig.set_size_inches(8, 3, forward=True) fig.set_size_inches(8, 3, forward=True)
if export: if export:
fig.savefig(file_name) fig.savefig(file_name)
print("Outliers:",len(outliers),";","Threshold[us]:",tresh) print("Outliers:", len(outliers), ";", "Threshold[us]:", tresh)
def trace_jitter(df_filtered, export=False, file_name=None): def trace_jitter(df_filtered, export=False, file_name=None):
thresh = get_outlier_treshold(df_filtered["EndToEndTime"].describe()) thresh = get_outlier_threshold(df_filtered["EndToEndTime"].describe())
df_no_outliers = df_filtered[df_filtered["EndToEndTime"] <= thresh] df_no_outliers = df_filtered[df_filtered["EndToEndTime"] <= thresh]
box(df_no_outliers, export, file_name) box(df_no_outliers, export, file_name)
print("{} / {} are no outliers.".format(len(df_no_outliers), len(df_filtered))) print("{} / {} are no outliers.".format(len(df_no_outliers), len(df_filtered)))
...@@ -60,6 +64,7 @@ def trace_jitter(df_filtered, export=False, file_name=None): ...@@ -60,6 +64,7 @@ def trace_jitter(df_filtered, export=False, file_name=None):
fig.canvas.set_window_title('Jitter Analysis') fig.canvas.set_window_title('Jitter Analysis')
plt.show() plt.show()
def prep(df, config): def prep(df, config):
columns = extract_durations(config) columns = extract_durations(config)
df_filtered = df[columns] df_filtered = df[columns]
......
...@@ -5,86 +5,96 @@ import collections ...@@ -5,86 +5,96 @@ import collections
from operator import itemgetter from operator import itemgetter
from sklearn import datasets, linear_model from sklearn import datasets, linear_model
def _extract_duration_by_src(all_durations, src): def _extract_duration_by_src(all_durations, src):
return [c for c,v in all_durations.items() if v["Source"] == src] return [c for c, v in all_durations.items() if v["Source"] == src]
def _stamp_name_by_src_and_type(all_stamps, src, kind=None): def _stamp_name_by_src_and_type(all_stamps, src, kind=None):
if kind is None: if kind is None:
kind = ["time", "cycle"] kind = ["time", "cycle"]
columns = [c for c,v in all_stamps.items() if v["Source"] == src and v["Type"] in kind] columns = [c for c, v in all_stamps.items() if v["Source"] == src and v["Type"] in kind]
return columns return columns
def _extract_stamps_by_type(all_stamps, src, kind=None): def _extract_stamps_by_type(all_stamps, src, kind=None):
columns = _stamp_name_by_src_and_type(all_stamps, src, kind) columns = _stamp_name_by_src_and_type(all_stamps, src, kind)
return _stamp_names(columns) return _stamp_names(columns)
def _stamp_names(columns): def _stamp_names(columns):
stamps = [x + "_T" for x in columns] + [x + "_C" for x in columns] stamps = [x + "_T" for x in columns] + [x + "_C" for x in columns]
return stamps return stamps
def _evaluate_file(fileName, stamps, kind, sender=False):
def _evaluate_file(file_name, stamps, kind, sender=False):
# Remove first line, as this is the dummy line for intermittently storing data. # Remove first line, as this is the dummy line for intermittently storing data.
df = pd.read_csv(fileName)[1:] df = pd.read_csv(file_name)[1:]
df = df[df["Kind"] == kind].drop(["Kind"], axis=1).set_index("SeqNo") df = df[df["Kind"] == kind].drop(["Kind"], axis=1).set_index("SeqNo")
if sender: if sender:
df.drop(_extract_stamps_by_type(stamps,"receiver"),axis=1,inplace=True) df.drop(_extract_stamps_by_type(stamps, "receiver"), axis=1, inplace=True)
else: else:
df.drop(_extract_stamps_by_type(stamps,"sender"),axis=1,inplace=True) df.drop(_extract_stamps_by_type(stamps, "sender"), axis=1, inplace=True)
df = df[pd.notnull(df).all(axis=1)] df = df[pd.notnull(df).all(axis=1)]
return df return df
def _restore_timestamp(df, column, cycle_time, base_c, base_t):
df[column + "_T"] = ((df[column + "_C"] - base_c) * df[cycle_time + "Cycle_D"] + base_t).astype(int)
def _diff_t_c(df, name, start, stop): def _diff_t_c(df, name, start, stop):
time = df[stop + "_T"] - df[start + "_T"] time = df[stop + "_T"] - df[start + "_T"]
cycles = (df[stop + "_C"] - df[start + "_C"]) cycles = (df[stop + "_C"] - df[start + "_C"])
return (time.astype(float), cycles.astype(float)) return time.astype(float), cycles.astype(float)
def _generate_processing_durations(df, name, start, stop):
time, cycles = _diff_t_c(df, name, start, stop)
df[name + "_D"] = time
df[name + "_C"] = cycles
def _generate_cycle_time(df, name, start, stop): def _generate_cycle_time(df, name, start, stop):
time, cycles = _diff_t_c(df, name, start, stop) time, cycles = _diff_t_c(df, name, start, stop)
df[name + "Cycle_D"] = time / cycles df[name + "Cycle_D"] = time / cycles
def _generate_duration(df, name, start, stop, cycleTimeColumn):
def _generate_duration(df, name, start, stop, cycle_time_column):
diff = df[stop + "_C"] - df[start + "_C"] diff = df[stop + "_C"] - df[start + "_C"]
df[name + "Cycles"] = diff df[name + "Cycles"] = diff
df[name + "Time"] = diff * df[cycleTimeColumn + "Cycle_D"] df[name + "Time"] = diff * df[cycle_time_column + "Cycle_D"]
def evaluate(sender_file, receiver_file, config, kind=0): def evaluate(sender_file, receiver_file, config, kind=0):
df1 = _evaluate_file(sender_file,config["stamps"], kind,True) stamps = config["stamps"]
df2 = _evaluate_file(receiver_file,config["stamps"], kind) df1 = _evaluate_file(sender_file, stamps, kind, True)
df2 = _evaluate_file(receiver_file, stamps, kind)
df = df1.join(df2) df = df1.join(df2)
# Processing Times and Cycle Durations tr = config["time_reference"]
_generate_processing_durations(df, "Sender", "PrrtSendStart", "LinkTransmitEnd") cr = config["cycle_reference"]
_generate_cycle_time(df, "Sender", "PrrtSendStart", "LinkTransmitEnd")
_generate_processing_durations(df, "Receiver", "LinkReceive", "PrrtDeliver") # Determine Channel Duration
_generate_cycle_time(df, "Receiver", "PrrtReceivePackage", "PrrtDeliver") df["Channel_D"] = df[tr["receiver"]["Start"] + "_T"] - df[tr["sender"]["Stop"] + "_T"]
df["ChannelTime"] = df["LinkReceive_T"] - df["LinkTransmitEnd_T"] # Correlate Receiver Timestamps with Sender Timestamps (subtracting Channel Duration)
df["EndToEndTime"] = df["Sender_D"] + df["Receiver_D"] for s in _stamp_name_by_src_and_type(stamps, "receiver", kind=["time"]):
df[s + "_T"] -= df["Channel_D"]
for src in ["sender", "receiver"]:
# Generate Processing Duration
src_name = src.capitalize()
# Correlate Receiver Times with Sender Times time, cycles = _diff_t_c(df, src_name, tr[src]["Start"], tr[src]["Stop"])
df["LinkReceive_T"] -= df["ChannelTime"] df[src_name + "_D"] = time
df["PrrtReceivePackage_T"] -= df["ChannelTime"] df[src_name + "_C"] = cycles
df["PrrtDeliver_T"] -= df["ChannelTime"]
# Generate Durations # Generate Cycle Times
for d, duration in config["durations"].items(): _generate_cycle_time(df, src_name, cr[src]["Start"], cr[src]["Stop"])
_generate_duration(df, d, duration["Start"], duration["Stop"], duration["Source"].capitalize())
# Recreate missing timestamps from cycles # Recreate missing timestamps from cycles
for stamp in _stamp_name_by_src_and_type(config["stamps"], "sender", "cycle"): for stamp_name in _stamp_name_by_src_and_type(stamps, src, "cycle"):
_restore_timestamp(df, stamp, "Sender", df["PrrtSendStart_C"], df["PrrtSendStart_T"]) start_stamp = tr[src]["Start"]
diff = df[stamp_name + "_C"] - df[start_stamp + "_C"]
df[stamp_name + "_T"] = (diff * df[src_name + "Cycle_D"] + df[start_stamp + "_T"]).astype(int)
for stamp in _stamp_name_by_src_and_type(config["stamps"], "receiver", "cycle"): # Generate Durations
_restore_timestamp(df, stamp, "Receiver", df["LinkReceive_C"], df["LinkReceive_T"]) for name, duration in config["durations"].items():
diff = df[duration["Stop"] + "_C"] - df[duration["Start"] + "_C"]
df[name + "Cycles"] = diff
df[name + "_D"] = diff * df[duration["Source"].capitalize() + "Cycle_D"]
df["EndToEndTime"] = df["Sender_D"] + df["Receiver_D"]
return df return df
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment