support to change loss dynamically in gym (#13)

* support change loss rate dynamically in gym and make a test script

* bugfix : error when handling trace with no loss and update related test script

* remove hardcode in the test script about loss rate

* optimized the test scripts about loss rate

1. support to set run_times to control the times that the trace is run
2. support to set bandwidth_bps to control send rate
3. reduced error rate indicators

alphartc_gym/tests/data/loss/trace_loss_0.json

@@ -0,0 +1,16 @@
+{
+    "type": "video",
+    "downlink": {},
+    "uplink": {
+        "trace_pattern": [
+            {
+                "duration": 200,
+                "capacity": 1000,
+                "loss": 0,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            }
+        ]
+    }
+}

alphartc_gym/tests/data/loss/trace_loss_0dot1.json

@@ -0,0 +1,16 @@
+{
+    "type": "video",
+    "downlink": {},
+    "uplink": {
+        "trace_pattern": [
+            {
+                "duration": 200,
+                "capacity": 1000,
+                "loss": 0.1,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            }
+        ]
+    }
+}

alphartc_gym/tests/data/loss/trace_loss_0dot5.json

@@ -0,0 +1,16 @@
+{
+    "type": "video",
+    "downlink": {},
+    "uplink": {
+        "trace_pattern": [
+            {
+                "duration": 200,
+                "capacity": 1000,
+                "loss": 0.5,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            }
+        ]
+    }
+}

alphartc_gym/tests/data/loss/trace_loss_pattern_2.json

@@ -0,0 +1,24 @@
+{
+    "type": "video",
+    "downlink": {},
+    "uplink": {
+        "trace_pattern": [
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            },
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0.2,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            }
+        ]
+    }
+}

alphartc_gym/tests/data/loss/trace_loss_pattern_3.json

@@ -0,0 +1,32 @@
+{
+    "type": "video",
+    "downlink": {},
+    "uplink": {
+        "trace_pattern": [
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            },
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0.2,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            },
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0.1,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            }
+        ]
+    }
+}

alphartc_gym/tests/data/loss/trace_loss_pattern_4.json

@@ -0,0 +1,40 @@
+{
+    "type": "video",
+    "downlink": {},
+    "uplink": {
+        "trace_pattern": [
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            },
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0.2,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            },
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0.5,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            },
+            {
+                "duration": 500,
+                "capacity": 1000,
+                "loss": 0.2,
+                "rtt": 0,
+                "jitter": 0,
+                "time": 0.0
+            }
+        ]
+    }
+}

alphartc_gym/tests/test_gym_loss.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from alphartc_gym import gym
+import os, json
+
+
+ERROR_STATIC = 0.05
+ERROR_DYNAMICAL = 0.07
+
+
+def get_gym_stats(trace_path, duration_time_ms=3000, bandwidth_bps=1000000):
+    total_stats = []
+    g = gym.Gym()
+    g.reset(trace_path=trace_path, duration_time_ms=duration_time_ms)
+
+    while True:
+        stats, done = g.step(bandwidth_bps)
+        if not done:
+            total_stats += stats
+        else:
+            return total_stats
+
+
+def get_info_from_trace(trace_file):
+    with open(trace_file, 'r') as f:
+        return json.loads(f.read())
+
+
+def get_abs_path_by_name(trace_name):
+    trace_path = os.path.join(
+            os.path.dirname(__file__),
+            "data/loss",
+            trace_name)
+    return trace_path
+
+
+def single_loss_available(trace_path):
+    total_stats = get_gym_stats(trace_path)
+    assert (total_stats)
+
+    for stats in total_stats:
+        assert (isinstance(stats, dict))
+
+
+def single_loss_persistence(trace_path, run_times=1, bandwidth_bps=1000000):
+    # get information from trace
+    trace_data = get_info_from_trace(trace_path)
+    trace_pattern = trace_data["uplink"]["trace_pattern"]
+    predict_error_rate = trace_pattern[0]["loss"]
+    duration_time_ms = sum([item["duration"] for item in trace_pattern]) * run_times
+
+    total_stats = get_gym_stats(trace_path, duration_time_ms=duration_time_ms, bandwidth_bps=bandwidth_bps)
+    assert (total_stats)
+
+    mp_src_seq = {}
+    now_total, now_loss = 0, 0
+    for i in range(len(total_stats)):
+        ssrc = total_stats[i]["ssrc"]
+        if ssrc not in mp_src_seq:
+            mp_src_seq[ssrc] = 0
+        # calculate loss packet
+        if mp_src_seq[ssrc] + 1 < total_stats[i]["sequence_number"]:
+            while mp_src_seq[ssrc] + 1 < total_stats[i]["sequence_number"]:
+                now_total += 1
+                now_loss += 1
+                mp_src_seq[ssrc] += 1
+        now_total += 1
+        mp_src_seq[ssrc] += 1
+    assert abs(now_loss / now_total - predict_error_rate) <= predict_error_rate * ERROR_STATIC
+
+
+def single_loss_dynamically(trace_path, run_times=1, bandwidth_bps=1000000):
+    # get information from trace
+    trace_data = get_info_from_trace(trace_path)
+    trace_pattern = trace_data["uplink"]["trace_pattern"]
+    trace_duration_time_ms = sum([item["duration"] for item in trace_pattern])
+    predict_error_rate = sum([item["loss"] * (item["duration"] / trace_duration_time_ms) for item in trace_pattern])
+    duration_time_ms = sum([item["duration"] for item in trace_pattern]) * run_times
+
+    total_stats = get_gym_stats(trace_path, duration_time_ms=duration_time_ms, bandwidth_bps=bandwidth_bps)
+    assert (total_stats)
+
+    mp_src_seq = {}
+    now_total, now_loss = 0, 0
+    for i in range(len(total_stats)):
+        ssrc = total_stats[i]["ssrc"]
+        if ssrc not in mp_src_seq:
+            mp_src_seq[ssrc] = 0
+        # calculate loss packet
+        if mp_src_seq[ssrc] + 1 < total_stats[i]["sequence_number"]:
+            while mp_src_seq[ssrc] + 1 < total_stats[i]["sequence_number"]:
+                now_total += 1
+                now_loss += 1
+                mp_src_seq[ssrc] += 1
+        now_total += 1
+        mp_src_seq[ssrc] += 1
+    assert abs(now_loss / now_total - predict_error_rate) <= predict_error_rate * ERROR_DYNAMICAL
+
+
+def test_loss_available():
+    traces_name = ["trace_loss_0.json", "trace_loss_0dot1.json", "trace_loss_0dot5.json"]
+    for trace in traces_name:
+        trace_path = get_abs_path_by_name(trace)
+        single_loss_available(trace_path)
+
+
+def test_loss_persistence():
+    traces_name = ["trace_loss_0.json", "trace_loss_0dot1.json", "trace_loss_0dot5.json"]
+    for trace in traces_name:
+        trace_path = get_abs_path_by_name(trace)
+        single_loss_persistence(trace_path, run_times=1000, bandwidth_bps=300000)
+
+def test_loss_dynamically():
+    traces_name = ["trace_loss_pattern_2.json", "trace_loss_pattern_3.json", "trace_loss_pattern_4.json"]
+    for trace in traces_name:
+        trace_path = get_abs_path_by_name(trace)
+        single_loss_dynamically(trace_path, run_times=1000, bandwidth_bps=300000)

ns-app/scratch/webrtc_test/trace_player.cc

@@ -3,6 +3,9 @@
 #include "ns3/point-to-point-net-device.h"
 #include "ns3/simulator.h"
 #include "ns3/string.h"
+#include "ns3/error-model.h"
+#include "ns3/double.h"
+#include "ns3/pointer.h"
 #include "ns3/channel.h"
 
 #include <nlohmann/json.hpp>
@@ -42,6 +45,9 @@ void TracePlayer::LoadTrace() {
         TraceItem ti;
         ti.capacity_ = lexical_cast<decltype(ti.capacity_)>(trace["capacity"]);
         ti.duration_ms_ = lexical_cast<decltype(ti.duration_ms_)>(trace["duration"]);
+        if (trace.find("loss") != trace.end()) {
+            ti.loss_rate_ = lexical_cast<double>(trace["loss"]);
+        }
         if (trace.find("rtt") != trace.end()) {
             ti.rtt_ms_ = lexical_cast<std::uint64_t>(trace["rtt"]);
         }        
@@ -69,6 +75,13 @@ void TracePlayer::PlayTrace(size_t trace_index) {
                 dynamic_cast<PointToPointNetDevice *>(PeekPointer(node->GetDevice(j)));
             if (device) {
                 device->SetDataRate(DataRate(trace.capacity_ * 1e3));
+                // set loss rate in every device
+                if (trace.loss_rate_) {
+                    Ptr<RateErrorModel> em = CreateObjectWithAttributes<RateErrorModel> ("RanVar", StringValue("ns3::UniformRandomVariable[Min=0.0|Max=1.0]"), \
+                                                                                     "ErrorRate", DoubleValue (trace.loss_rate_.value()), \
+                                                                                     "ErrorUnit", StringValue("ERROR_UNIT_PACKET"));
+                    device->SetAttribute("ReceiveErrorModel", PointerValue (em));
+                }
             }
         }
     }

ns-app/scratch/webrtc_test/trace_player.h

@@ -10,6 +10,7 @@
 struct TraceItem {
     std::uint64_t capacity_;
     std::uint64_t duration_ms_;
+    boost::optional<double> loss_rate_;
     boost::optional<std::uint64_t> rtt_ms_;
 };