Commit 8e129984 authored by Ashkan's avatar Ashkan

Changes to run tests on input data predicted from ml model.

parent 4b8e346d
Pipeline #4524 failed with stages
in 26 seconds
import fec_balance_utils
fec_balance_utils_obj = fec_balance_utils.FecBalanceUtils("/home/ashkan/Documents/deep-hec/code/test_input_with_predicted_fec_balance")
fec_balance_utils_obj = fec_balance_utils.FecBalanceUtils("/home/ashkan/Documents/deep-hec/data/input_predicted_fec_balance")
# ri = fec_balance_utils_obj.get_ri_single_input(35)
# print(ri)
fec_balance_utils_obj.get_all_ri()
\ No newline at end of file
fec_balance_utils_obj.get_all_ri()
# conf = ric.gen_repair_schedule(3, 10, 1, 1)
# print(conf)
\ No newline at end of file
import math
import prrt
import time
import numpy as np
import pandas as pd
import prrt_utils
......@@ -20,22 +21,25 @@ class FecBalanceUtils:
self.columns_order = ["app_max_latency", "app_max_residual_loss_rate", "app_data_rate", "app_pkt_length",
"ch_loss_rate", "ch_rtt_prop_fwd", "ch_data_rate_btl_fwd",
"sys_red_pkt_trans_dly", "sys_src_pkt_interval",
"fec_balance", "config", "ri"]
"fec_balance", "config", "time", "ri"]
print("FecBalanceUtils Class is initialized")
def get_arq_delay(self, index):
d_t = self.predicted_table_in_df["app_max_latency"][index]
fec_balance = self.predicted_table_in_df["fec_balance"][index]
return (1 - fec_balance) * d_t
def get_nc(self, index):
arq_delay = self.get_arq_delay(index)
ch_rtt_prop_fwd = self.predicted_table_in_df["ch_rtt_prop_fwd"][index]
sys_red_pkt_trans_dly = self.predicted_table_in_df["sys_red_pkt_trans_dly"][index]
nc = math.ceil(arq_delay / (ch_rtt_prop_fwd + sys_red_pkt_trans_dly))
nc = math.floor(arq_delay / (ch_rtt_prop_fwd + sys_red_pkt_trans_dly))
return nc
def get_k(self, index):
d_t = self.predicted_table_in_df["app_max_latency"][index]
ch_rtt_prop_fwd = self.predicted_table_in_df["ch_rtt_prop_fwd"][index]
......@@ -43,16 +47,14 @@ class FecBalanceUtils:
sys_src_pkt_interval = self.predicted_table_in_df["sys_src_pkt_interval"][index]
self.nc = min(self.get_nc(index), self.nc_cap)
k = (d_t - ch_rtt_prop_fwd - self.nc * (ch_rtt_prop_fwd + sys_red_pkt_trans_dly)) / max(sys_red_pkt_trans_dly, sys_src_pkt_interval)
return math.ceil(k)
return math.floor(k)
def get_k_lim(self, index):
get_k_lim = prrt_utils.get_k_lim(self.n_min, self.n_max, self.predicted_table_in_df["ch_loss_rate"][index],
self.predicted_table_in_df["app_max_residual_loss_rate"][index])
return get_k_lim
def get_ri_single_input(self, index):
self.k = min(self.get_k_lim(index), self.get_k(index))
k_lim = prrt_utils.get_k_lim(self.n_min, self.n_max, self.predicted_table_in_df["ch_loss_rate"][index],
self.predicted_table_in_df["app_max_residual_loss_rate"][index])
k_loss = self.get_k(index)
self.k = min(k_lim, k_loss)
ch_rtt_prop_fwd = self.predicted_table_in_df["ch_rtt_prop_fwd"][index]
ch_data_rate_btl_fwd = self.predicted_table_in_df["ch_data_rate_btl_fwd"][index]
ch_loss_rate = self.predicted_table_in_df["ch_loss_rate"][index]
......@@ -62,7 +64,7 @@ class FecBalanceUtils:
n_p_max = prrt_utils.get_n_p_max(ch_rtt_prop_fwd, app_pkt_length, ch_data_rate_btl_fwd)
self.n = prrt_utils.estimate_n_for_k(self.k, min(self.n_max, self.nc * n_p_max), ch_loss_rate, app_max_residual_loss_rate)
if (self.n - self.k) * self.n_p_min < self.nc or n_p_max == 0:
if n_p_max == 0:
self.repair_schedule = []
return -1
......@@ -90,12 +92,17 @@ class FecBalanceUtils:
return coding_conf.get_redundant_information()
def get_all_ri(self):
save_result_to = pd.DataFrame()
total_start = time.time()
for index, row in self.predicted_table_in_df.iterrows():
if index < 150:
if index < 200:
print(index)
start_time = time.time()
ri = self.get_ri_single_input(index)
duration = time.time() - start_time
# print(duration)
save_result_to = save_result_to.append({'app_max_latency': row['app_max_latency'],
'app_max_residual_loss_rate': row['app_max_residual_loss_rate'],
'app_data_rate': row['app_data_rate'],
......@@ -106,7 +113,9 @@ class FecBalanceUtils:
'sys_red_pkt_trans_dly': np.round(row['sys_red_pkt_trans_dly'], 5),
'sys_src_pkt_interval': row['sys_src_pkt_interval'],
'fec_balance': row['fec_balance'],
'config': [self.k, self.n, self.nc, self.repair_schedule],
'config': [self.k, self.n, self.repair_schedule, self.nc],
'time' : duration,
'ri':ri}, ignore_index=True, sort=False)
save_result_to.to_csv('ri_from_predicted_model', sep=',', index=False, columns = self.columns_order)
\ No newline at end of file
save_result_to.to_csv('/home/ashkan/Documents/deep-hec/data/input_predicted_fec_balance_ml_result', sep=',', columns = self.columns_order)
print(time.time() - total_start)
\ No newline at end of file
......@@ -27,7 +27,7 @@ class HECSearch:
n_c_cap = 10
fec_balance = -1
n_c_max = math.ceil((self.prrtApplicationParameters.max_latency - self.get_fec_delay_min()) /
n_c_max = math.floor((self.prrtApplicationParameters.max_latency - self.get_fec_delay_min()) /
(self.prrtChannelParameters.rtt_prop_fwd +
self.prrtSystemParameters.redundancy_packet_transmission_delay))
n_c_max = min(n_c_max, n_c_cap)
......@@ -98,10 +98,10 @@ class HECSearch:
# print("k_opt=" + str(k_opt) + " n_opt=" + str(n_opt) + " n_p_opt=" + str(n_p_opt) + "RI= " + str(ri_opt))
if len(n_p_opt) != 0:
fec_balance = self.get_balance_delay_fec(n_p_opt[0], k_opt)
return [k_opt, n_opt, n_p_opt, fec_balance]
return [k_opt, n_opt, n_p_opt, n_c, fec_balance, ri_opt]
def get_k(self, n_c, arq_delay_i):
return math.ceil((self.prrtApplicationParameters.max_latency -
return math.floor((self.prrtApplicationParameters.max_latency -
self.prrtChannelParameters.rtt_prop_fwd -
n_c * arq_delay_i) / max(self.prrtSystemParameters.redundancy_packet_transmission_delay, self.prrtSystemParameters.source_packet_interval))
......
......@@ -11,27 +11,27 @@ from multiprocessing import Pool
# --------------------------------------------------------------
# Generate dataset
ds_all_input = set(os.listdir("/home/ashkan/Documents/hecps/code/ML/data_left_in_small_ds/ds17/")) # TODO: Adjust dataset dir
print(len(ds_all_input))
ds_calculated_output = set(os.listdir("documents/output/out_ds17"))
ds_calculated_output = {s.replace("out", "in") for s in ds_calculated_output}
print(len(ds_calculated_output))
ds_basename = list(ds_all_input - ds_calculated_output)
print(len(ds_basename))
ds_rel_input_path = "/home/ashkan/Documents/hecps/code/ML/data_left_in_small_ds/ds17/" # TODO: Adjust dataset dir
ds_rel_output_path = "documents/output/out_ds17"
# ds_all_input = set(os.listdir("/home/ashkan/Documents/hecps/code/ML/data_left_in_small_ds/ds17/")) # TODO: Adjust dataset dir
# print(len(ds_all_input))
# ds_calculated_output = set(os.listdir("documents/output/out_ds17"))
# ds_calculated_output = {s.replace("out", "in") for s in ds_calculated_output}
# print(len(ds_calculated_output))
# ds_basename = list(ds_all_input - ds_calculated_output)
# print(len(ds_basename))
# ds_rel_input_path = "/home/ashkan/Documents/hecps/code/ML/data_left_in_small_ds/ds17/" # TODO: Adjust dataset dir
# ds_rel_output_path = "documents/output/out_ds17"
#----------------------------------------------------------------
# ds_basename = os.listdir("documents/input/")
# ds_basename = "in_12_param_4_sz_zzbodx"
# ds_rel_input_path = "documents/input/"
# ds_rel_output_path = "documents/output/"
# TODO: Adjust dataset dir
ds_basename = os.listdir("/home/ashkan/Documents/deep-hec/data/")
ds_basename = "input_predicted_fec_balance"
ds_rel_input_path = "/home/ashkan/Documents/deep-hec/data/"
ds_rel_output_path = "/home/ashkan/Documents/deep-hec/data/"
columns_order = ["app_max_latency", "app_max_residual_loss_rate", "app_data_rate", "app_pkt_length",
"ch_loss_rate", "ch_rtt_prop_fwd", "ch_data_rate_btl_fwd",
"sys_red_pkt_trans_dly", "sys_src_pkt_interval",
"time", "config", "fec_balance"]
"fec_balance","config", "time", "ri"]
def evaluate(searchAlgorithm, appParams, channelParams, systemParams):
......@@ -49,10 +49,12 @@ def evaluate(searchAlgorithm, appParams, channelParams, systemParams):
def test_case(dataset_basename):
save_result_to = pd.DataFrame()
# Load dataset in chunk
total_time = time.time()
for df_in_chunk in pd.read_csv(ds_rel_input_path + dataset_basename, sep=',', chunksize=100):
print(dataset_basename + " started.")
# print(dataset_basename + " started.")
for index, row in df_in_chunk.iterrows():
if index != -1:
if index < 200:
print(index)
appParams = prrt.PrrtApplicationParameters(row['app_max_latency'], row['app_max_residual_loss_rate'], row['app_data_rate'], row['app_pkt_length'])
chnlParams = prrt.PrrtChannelParameters(row['ch_loss_rate'], 0, row['ch_rtt_prop_fwd'], 0, row['ch_data_rate_btl_fwd'], 0)
sysParams = prrt.PrrtSystemParameters(0, np.round(row['sys_red_pkt_trans_dly'], 5), 0, 0, row['sys_src_pkt_interval'])
......@@ -69,9 +71,10 @@ def test_case(dataset_basename):
'ch_data_rate_btl_fwd' : row['ch_data_rate_btl_fwd'],
'sys_red_pkt_trans_dly' : np.round(row['sys_red_pkt_trans_dly'], 5),
'sys_src_pkt_interval' : row['sys_src_pkt_interval'],
'time': np.around(search_result[1], 0),
'config': [config[0], config[1], config[2]],
'fec_balance': config[3]}, ignore_index=True, sort=False)
'fec_balance': config[4],
'config': [config[0], config[1], config[2], config[3]],
'time': search_result[1],
'ri' : config[5]}, ignore_index=True, sort=False)
else:
save_result_to = save_result_to.append({'app_max_latency': row['app_max_latency'],
'app_max_residual_loss_rate': row['app_max_residual_loss_rate'],
......@@ -82,22 +85,24 @@ def test_case(dataset_basename):
'ch_data_rate_btl_fwd' : row['ch_data_rate_btl_fwd'],
'sys_red_pkt_trans_dly' : np.round(row['sys_red_pkt_trans_dly'], 5),
'sys_src_pkt_interval' : row['sys_src_pkt_interval'],
# 'search': searchAlgorithm,
'time' : 0,
'fec_balance': -1,
'config': ["INV_PRM_NPM"],
'fec_balance': -1}, ignore_index=True, sort=False)
'time' : 0,
'ri' : 0}, ignore_index=True, sort=False)
save_result_to.to_csv(ds_rel_output_path + dataset_basename.replace("in", "out", 1), sep=',', index = False, columns = columns_order)
print(dataset_basename + " finished.")
# save_result_to.to_csv(ds_rel_output_path + dataset_basename.replace("in", "out", 1), sep=',', index = False, columns = columns_order)
save_result_to.to_csv("/home/ashkan/Documents/deep-hec/data/input_predicted_fec_balance_greedy_result", sep=',', columns=columns_order)
print(time.time() - total_time)
# print(dataset_basename + " finished.")
#--- for testing and profiling ---#
# test_case(ds_basename)
test_case(ds_basename)
#--- Freeflow --- #
if __name__ == '__main__':
pool = Pool(processes=7) # TODO: Adjust number of cores
pool.map(test_case, ds_basename, chunksize=1)
# if __name__ == '__main__':
# pool = Pool(processes=7) # TODO: Adjust number of cores
# pool.map(test_case, ds_basename, chunksize=1)
#--- scheduling --- #
# if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment