From fc1b6ab96707245dc87a39208459284c8edb8d5b Mon Sep 17 00:00:00 2001 From: wangyf2001 <392019123@qq.com> Date: Thu, 22 Dec 2022 02:29:20 +0000 Subject: [PATCH 01/12] std-app-process --- .gitignore | 104 ++++++++++++++++++++++ Makefile | 9 ++ azure-pipelines.yml | 41 +++++++++ bwmur/bwmeasure.py | 135 +++++++++++++++++++++++++++++ bwmur/bwmurclient.py | 91 +++++++++++++++++++ bwmur/bwmursever.py | 91 +++++++++++++++++++ bwmur/tests/.test_bwmeasure.py.swp | Bin 0 -> 12288 bytes bwmur/tests/clientbwlog.txt | 19 ++++ bwmur/tests/machines.json | 16 ++++ bwmur/tests/severbwlog.txt | 20 +++++ bwmur/tests/test_bwmeasure.py | 33 +++++++ dockers/Dockerfile | 15 ++++ 12 files changed, 574 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 azure-pipelines.yml create mode 100644 bwmur/bwmeasure.py create mode 100644 bwmur/bwmurclient.py create mode 100644 bwmur/bwmursever.py create mode 100644 bwmur/tests/.test_bwmeasure.py.swp create mode 100644 bwmur/tests/clientbwlog.txt create mode 100644 bwmur/tests/machines.json create mode 100644 bwmur/tests/severbwlog.txt create mode 100644 bwmur/tests/test_bwmeasure.py create mode 100644 dockers/Dockerfile diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..894a44c --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..665441f --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +docker_image := net-env +docker_file := dockers/Dockerfile + +all: net-env + +net-env: + docker pull ubuntu:20.04 + docker build . --build-arg UID=$(shell id -u) --build-arg GUID=$(shell id -g) -f $(docker_file) -t ${docker_image} + diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..1ed0807 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,41 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: + branches: + include: + - "*" + exclude: + - upstream/* + + +pool: + vmImage: 'ubuntu-latest' + +steps: + +- checkout: self + +- script: make + displayName: 'build network measure environment' + +- script: cd /home ; sudo wget https://github.com/OpenNetLab/OpenNetLab-P2P-Measurment/blob/v-wangyif/bwpitch/machines.json + displayName: 'make network measure environment' + +- script: docker run -v /home:/app -w /app --net=host --name net_evals net-env:latest python3 /home/onl/bwmur/bwmursever.py && docker run -v /home:/app -w /app --net=host --name net_evalc net-env:latest python3 /home/onl/bwmur/bwmurclient.py + displayName: 'Run test' + + +- script: docker image tag net-env:latest opennetlab.azurecr.io/net-env:latest + displayName: 'Tag network environment image' + +- task: Docker@2 + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + inputs: + containerRegistry: 'opennetlab Azure registry' + repository: 'net-env' + command: 'push' + tags: 'latest' + displayName: "Push net-env image" diff --git a/bwmur/bwmeasure.py b/bwmur/bwmeasure.py new file mode 100644 index 0000000..ce62b10 --- /dev/null +++ b/bwmur/bwmeasure.py @@ -0,0 +1,135 @@ +import paramiko +import time +import random +import json +import os +import sys +from scp import SCPClient +import numpy as np +import copy +import time + +socket_timeout_sec = 240 +machines_file = "machines.json" +sever_port = "8000" +sever_ip = "20.81.187.38" +recv_wd = "/home/wang/testnet/" + +sever_run_cmd = [ + "iperf3 -s -p %s -i 1" % ( + sever_port)] +client_run_cmd = [ + "iperf3 -c %s -p %s" % ( + sever_ip, sever_port)] + + +def get_datetime(): + now = int(round(time.time() * 1000)) + ret = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now / 1000)) + return ret + + +def get_ssh(name): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + client.connect(hostname=machines[name]["host"], port=machines[name]["ssh_port"], username=machines[name]["user"], + password=machines[name]["pwd"]) + + #print(machines[name]["host"]) + #print(machines[name]["ssh_port"]) + #print(machines[name]["user"]) + + return client + +def get_bw(name): + global sever_ip + global sever_port + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + #print("iperf") + #print(machines[name]["host"]) + #print(machines[name]["bw_port"]) + sever_ip = machines[name]["host"] + sever_port = machines[name]["bw_port"] + + + + + +def BWmeasure(matches_num): + #print(get_datetime(), "start measure") + try: + sender = get_ssh("sender_%d" % (matches_num)) + recv = get_ssh("recv_%d" % (matches_num)) + get_bw("recv_%d" % (matches_num)) + shell_sender, shell_recv = sender.invoke_shell(), recv.invoke_shell() + shell_sender.settimeout(socket_timeout_sec) + shell_recv.settimeout(socket_timeout_sec) + + # shell_sender.send("sudo su\n") + # shell_recv.send("sudo su\n") + # time.sleep(3) + output =" " + + def start_recv(): + #print(get_datetime(), "start recv") + cmd = sever_run_cmd + #print(' '.join(cmd)) + shell_recv.send(' '.join(cmd) + "\n") + # time.sleep(1000) + + def start_sender(): + output = " " + #print(get_datetime(), "start sender") + + cmd = client_run_cmd + #print(get_datetime(), ' '.join(cmd)) + shell_sender.send(' '.join(cmd) + "\n") + times=0 + doc = open('bwlog.txt', 'a+') + doc.seek(0) + doc.truncate() + doc.close() + while times < 15: + time.sleep(1) + times += 1 + if times <2: + if shell_sender.recv_ready(): + outbuf = shell_sender.recv(65535) + continue + if shell_sender.recv_ready(): + outbuf = shell_sender.recv(65535) + if len(outbuf) == 0: + break + output += outbuf.decode("utf-8", "ignore") + doc = open('bwlog.txt', 'a+') + doc.write(outbuf.decode("utf-8", "ignore")) + doc.close() + print(output) + start_recv() + time.sleep(1) + print("signal signal") + start_sender() + netctr = get_ssh("netctr") + print("start scp") + scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) + scp_client.put("bwlog.txt", "%s/." % (recv_wd)) + scp_client.close() + + + except Exception as e: + print(get_datetime(), "run_measure", e) + + finally: + print("measure finish") + +if __name__ == "__main__": + BWmeasure(1) diff --git a/bwmur/bwmurclient.py b/bwmur/bwmurclient.py new file mode 100644 index 0000000..058af48 --- /dev/null +++ b/bwmur/bwmurclient.py @@ -0,0 +1,91 @@ +import paramiko +import time +import random +import json +import os +import sys +from scp import SCPClient +import numpy as np +import copy +import time + +socket_timeout_sec = 240 +machines_file = "machines.json" +sever_port = "8000" +sever_ip = "127.0.0.1" +recv_wd = "/home" + +sever_run_cmd = [ + "iperf3 -s -p %s -i 1 -1" % ( + sever_port)] +client_run_cmd = [ + "iperf3 -c %s -p %s" % ( + sever_ip, sever_port)] + +def get_datetime(): + now = int(round(time.time() * 1000)) + ret = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now / 1000)) + return ret + + +def get_bw(name): + global sever_ip + global sever_port + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + #print("iperf") + #print(machines[name]["host"]) + #print(machines[name]["bw_port"]) + sever_ip = machines[name]["host"] + sever_port = machines[name]["bw_port"] + print(machines[name]["host"]) + print(sever_ip) + + + +def get_ssh(name): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + client.connect(hostname=machines[name]["host"], port=machines[name]["ssh_port"], username=machines[name]["user"], + password=machines[name]["pwd"]) + + + return client + +def BWmeasure(matches_num): + #print(get_datetime(), "start measure") + try: + output =" " + get_bw("recv_%d" % (matches_num)) + client_run_cmd = ["iperf3 -c %s -p %s" % (sever_ip, sever_port)] + status = os.popen(" ".join(client_run_cmd)).read() + print(" ".join(client_run_cmd)) + cmd = client_run_cmd + doc = open('clientbwlog.txt', 'a+') + doc.seek(0) + doc.truncate() + doc.write(status) + doc.close() + netctr = get_ssh("netctr") + print("start scp") + scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) + scp_client.put("clientbwlog.txt", "%s/." % (recv_wd)) + scp_client.close() + + + except Exception as e: + print(get_datetime(), "run_measure", e) + + finally: + print("measure finish") + +if __name__ == "__main__": + BWmeasure(1) diff --git a/bwmur/bwmursever.py b/bwmur/bwmursever.py new file mode 100644 index 0000000..205e7b4 --- /dev/null +++ b/bwmur/bwmursever.py @@ -0,0 +1,91 @@ +import paramiko +import time +import random +import json +import os +import sys +from scp import SCPClient +import numpy as np +import copy +import time + +socket_timeout_sec = 240 +machines_file = "machines.json" +sever_port = "8000" +sever_ip = "20.81.187.38" +recv_wd = "/home/wang/testnet/" + +sever_run_cmd = [ + "iperf3 -s -p %s -i 1 -1" % ( + sever_port)] +client_run_cmd = [ + "iperf3 -c %s -p %s" % ( + sever_ip, sever_port)] + +def get_datetime(): + now = int(round(time.time() * 1000)) + ret = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now / 1000)) + return ret + + +def get_bw(name): + global sever_ip + global sever_port + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + #print("iperf") + #print(machines[name]["host"]) + #print(machines[name]["bw_port"]) + sever_ip = machines[name]["host"] + sever_port = machines[name]["bw_port"] + + + +def get_ssh(name): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + client.connect(hostname=machines[name]["host"], port=machines[name]["ssh_port"], username=machines[name]["user"], + password=machines[name]["pwd"]) + + #print(machines[name]["host"]) + #print(machines[name]["ssh_port"]) + #print(machines[name]["user"]) + + return client + +def BWmeasure(matches_num): + #print(get_datetime(), "start measure") + try: + output =" " + get_bw("recv_%d" % (matches_num)) + sever_run_cmd = ["iperf3 -s -p %s -i 1 -1" % (sever_port)] + status = os.popen(" ".join(sever_run_cmd)).read() + cmd = sever_run_cmd + doc = open('severbwlog.txt', 'a+') + doc.seek(0) + doc.truncate() + doc.write(status) + doc.close() + netctr = get_ssh("netctr") + print("start scp") + scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) + scp_client.put("severbwlog.txt", "%s/." % (recv_wd)) + scp_client.close() + + + except Exception as e: + print(get_datetime(), "run_measure", e) + + finally: + print("measure finish") + +if __name__ == "__main__": + BWmeasure(1) diff --git a/bwmur/tests/.test_bwmeasure.py.swp b/bwmur/tests/.test_bwmeasure.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..26a796805bd7bca5c80d997e16fc8c6c061ce2c9 GIT binary patch literal 12288 zcmeI2&u`N(6vsUTC$xd!z+wFGvP*-Sv|0I4P4t4;&T?>*bkpPs6`r2FdN8SRG=fp(RUUtdnr zubUUijZfy77so}>iu#-#+&P=7b8&$gF7Mc1TFI{0dJ9QUy zAOR$BW&)MXgZ{>-w-a^U+HG%(UcWXv^ES8u2_OL^fCP{L5%Yvd!O zf1$a;jiz7wt8b>*Apsg|Kk7u zk1K?H2Ymy51|5SwfZl_ig7!gsp!=Xr&}Goc1|bUc8uSQs2Xq^B3v?5-4sz%GbZ22h z0!RP}AOR$R1dsp{Kmter2_S+0O@Km6$2@%-Co98g%O@GnsVm2ZLhP2m>+)~`%owgM z^&ShWndKf8l4i`Z+UW*#K+__NH8<77hQ?;9vbD3+LfY0JpI=*Hp28AOlw9XM8l*JZ zc^G!#iD+PH)b$o&&6W^ss-)B^<;H}%l5R`Wo?l`x)D>^j`Kg1IxWj|Pz1q&TUbVj* zK6x=3wkemX%7i=`cu*SK_vX1RGd9ay&1$ZO29Z~DY6v!<(n`mf&=9IjSTZ{fVJ2C@ zTZ=hMjI+1mI2RKh$G#7EbFB&>UX;#8MNujM{T!BMLoh1M^lTqIr12=Y^c$mOo0c>8 tJHhpqT-wx%f9e@+3f%T literal 0 HcmV?d00001 diff --git a/bwmur/tests/clientbwlog.txt b/bwmur/tests/clientbwlog.txt new file mode 100644 index 0000000..46e8df3 --- /dev/null +++ b/bwmur/tests/clientbwlog.txt @@ -0,0 +1,19 @@ +Connecting to host 127.0.0.1, port 8000 +[ 5] local 127.0.0.1 port 39098 connected to 127.0.0.1 port 8000 +[ ID] Interval Transfer Bitrate Retr Cwnd +[ 5] 0.00-1.00 sec 3.28 GBytes 28.1 Gbits/sec 0 3.12 MBytes +[ 5] 1.00-2.00 sec 4.28 GBytes 36.7 Gbits/sec 0 3.12 MBytes +[ 5] 2.00-3.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes +[ 5] 3.00-4.00 sec 4.29 GBytes 36.8 Gbits/sec 0 3.12 MBytes +[ 5] 4.00-5.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes +[ 5] 5.00-6.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes +[ 5] 6.00-7.00 sec 4.30 GBytes 36.9 Gbits/sec 0 3.12 MBytes +[ 5] 7.00-8.00 sec 4.26 GBytes 36.6 Gbits/sec 0 3.12 MBytes +[ 5] 8.00-9.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes +[ 5] 9.00-10.00 sec 4.28 GBytes 36.8 Gbits/sec 0 3.12 MBytes +- - - - - - - - - - - - - - - - - - - - - - - - - +[ ID] Interval Transfer Bitrate Retr +[ 5] 0.00-10.00 sec 41.8 GBytes 35.9 Gbits/sec 0 sender +[ 5] 0.00-10.04 sec 41.8 GBytes 35.7 Gbits/sec receiver + +iperf Done. diff --git a/bwmur/tests/machines.json b/bwmur/tests/machines.json new file mode 100644 index 0000000..5c9c30d --- /dev/null +++ b/bwmur/tests/machines.json @@ -0,0 +1,16 @@ +{ + + "recv_1" : { + "host" : "127.0.0.1", + "bw_port" : "8000", + "user" : "wang", + "pwd" : "123456789wang@" + }, + "netctr" : { + "host" : "127.0.0.1", + "ssh_port" : "22", + "user" : "root", + "pwd" : "" + } + +} diff --git a/bwmur/tests/severbwlog.txt b/bwmur/tests/severbwlog.txt new file mode 100644 index 0000000..d4d2a99 --- /dev/null +++ b/bwmur/tests/severbwlog.txt @@ -0,0 +1,20 @@ +----------------------------------------------------------- +Server listening on 8000 +----------------------------------------------------------- +Accepted connection from 127.0.0.1, port 39090 +[ 5] local 127.0.0.1 port 8000 connected to 127.0.0.1 port 39098 +[ ID] Interval Transfer Bitrate +[ 5] 0.00-1.00 sec 3.09 GBytes 26.5 Gbits/sec +[ 5] 1.00-2.00 sec 4.28 GBytes 36.8 Gbits/sec +[ 5] 2.00-3.00 sec 4.27 GBytes 36.6 Gbits/sec +[ 5] 3.00-4.00 sec 4.28 GBytes 36.8 Gbits/sec +[ 5] 4.00-5.00 sec 4.27 GBytes 36.7 Gbits/sec +[ 5] 5.00-6.00 sec 4.27 GBytes 36.7 Gbits/sec +[ 5] 6.00-7.00 sec 4.30 GBytes 36.9 Gbits/sec +[ 5] 7.00-8.00 sec 4.26 GBytes 36.6 Gbits/sec +[ 5] 8.00-9.00 sec 4.27 GBytes 36.7 Gbits/sec +[ 5] 9.00-10.00 sec 4.28 GBytes 36.8 Gbits/sec +[ 5] 10.00-10.04 sec 190 MBytes 36.7 Gbits/sec +- - - - - - - - - - - - - - - - - - - - - - - - - +[ ID] Interval Transfer Bitrate +[ 5] 0.00-10.04 sec 41.8 GBytes 35.7 Gbits/sec receiver diff --git a/bwmur/tests/test_bwmeasure.py b/bwmur/tests/test_bwmeasure.py new file mode 100644 index 0000000..ab01221 --- /dev/null +++ b/bwmur/tests/test_bwmeasure.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os, json, pytest, subprocess, time +from tempfile import NamedTemporaryFile + + +cur_dir = os.path.dirname(os.path.abspath(__file__)) + + +def check_bws(): + file_path = cur_dir + "/../bwmursever.py" + cmds = ["python3",file_path] + cmds = (" ".join(cmds)) + subprocess.Popen(cmds,shell = True) + #os.system(cmds) + + +def check_bwc(): + file_path = cur_dir + "/../bwmurclient.py" + cmdc = ["python3",file_path] + cmdc = (" ".join(cmdc)) + #cmd_result = subprocess.Popen(cmdc,shell = True).stdout.read() + cmd_result = os.popen(cmdc).read() + print(cmd_result) + assert "scp" in cmd_result + + + +def test_bw(): + check_bws() + print("OK") + check_bwc() diff --git a/dockers/Dockerfile b/dockers/Dockerfile new file mode 100644 index 0000000..51c2346 --- /dev/null +++ b/dockers/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:20.04 + +# Add ONL user +ARG USER=onl + +WORKDIR /home/${USER} + +ARG DEBIAN_FRONTEND=noninteractive +# Install dependency +RUN apt-get update && apt-get install -y \ + python3-pip wget unzip gocr imagemagick iperf3 + +RUN pip3 install pytest numpy requests soundfile paramiko scp + +COPY bwmur bwmur From 6ca92b6dacef2d834728ca8bdec4f213b7531003 Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Thu, 22 Dec 2022 10:41:13 +0800 Subject: [PATCH 02/12] Create machines.json --- machines.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 machines.json diff --git a/machines.json b/machines.json new file mode 100644 index 0000000..5c9c30d --- /dev/null +++ b/machines.json @@ -0,0 +1,16 @@ +{ + + "recv_1" : { + "host" : "127.0.0.1", + "bw_port" : "8000", + "user" : "wang", + "pwd" : "123456789wang@" + }, + "netctr" : { + "host" : "127.0.0.1", + "ssh_port" : "22", + "user" : "root", + "pwd" : "" + } + +} From fd0047b7dd91470a22df2b30fbb020f86a96b487 Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Thu, 22 Dec 2022 10:41:44 +0800 Subject: [PATCH 03/12] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1ed0807..75bbc3d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ steps: - script: make displayName: 'build network measure environment' -- script: cd /home ; sudo wget https://github.com/OpenNetLab/OpenNetLab-P2P-Measurment/blob/v-wangyif/bwpitch/machines.json +- script: cd /home ; sudo wget https://github.com/OpenNetLab/OpenNetLab-P2P-Measurment/blob/v-wangyif/std-process-pitch/machines.json displayName: 'make network measure environment' - script: docker run -v /home:/app -w /app --net=host --name net_evals net-env:latest python3 /home/onl/bwmur/bwmursever.py && docker run -v /home:/app -w /app --net=host --name net_evalc net-env:latest python3 /home/onl/bwmur/bwmurclient.py From fedbfa1c078637d615a8476d7c6b965850c176ab Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Tue, 27 Dec 2022 11:27:17 +0800 Subject: [PATCH 04/12] Delete bwmeasure.py --- bwmur/bwmeasure.py | 135 --------------------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 bwmur/bwmeasure.py diff --git a/bwmur/bwmeasure.py b/bwmur/bwmeasure.py deleted file mode 100644 index ce62b10..0000000 --- a/bwmur/bwmeasure.py +++ /dev/null @@ -1,135 +0,0 @@ -import paramiko -import time -import random -import json -import os -import sys -from scp import SCPClient -import numpy as np -import copy -import time - -socket_timeout_sec = 240 -machines_file = "machines.json" -sever_port = "8000" -sever_ip = "20.81.187.38" -recv_wd = "/home/wang/testnet/" - -sever_run_cmd = [ - "iperf3 -s -p %s -i 1" % ( - sever_port)] -client_run_cmd = [ - "iperf3 -c %s -p %s" % ( - sever_ip, sever_port)] - - -def get_datetime(): - now = int(round(time.time() * 1000)) - ret = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now / 1000)) - return ret - - -def get_ssh(name): - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - with open(machines_file, 'r') as f: - machines = json.loads(f.read()) - if name not in machines: - raise ValueError("Not find such mahcine") - - client.connect(hostname=machines[name]["host"], port=machines[name]["ssh_port"], username=machines[name]["user"], - password=machines[name]["pwd"]) - - #print(machines[name]["host"]) - #print(machines[name]["ssh_port"]) - #print(machines[name]["user"]) - - return client - -def get_bw(name): - global sever_ip - global sever_port - with open(machines_file, 'r') as f: - machines = json.loads(f.read()) - if name not in machines: - raise ValueError("Not find such mahcine") - - #print("iperf") - #print(machines[name]["host"]) - #print(machines[name]["bw_port"]) - sever_ip = machines[name]["host"] - sever_port = machines[name]["bw_port"] - - - - - -def BWmeasure(matches_num): - #print(get_datetime(), "start measure") - try: - sender = get_ssh("sender_%d" % (matches_num)) - recv = get_ssh("recv_%d" % (matches_num)) - get_bw("recv_%d" % (matches_num)) - shell_sender, shell_recv = sender.invoke_shell(), recv.invoke_shell() - shell_sender.settimeout(socket_timeout_sec) - shell_recv.settimeout(socket_timeout_sec) - - # shell_sender.send("sudo su\n") - # shell_recv.send("sudo su\n") - # time.sleep(3) - output =" " - - def start_recv(): - #print(get_datetime(), "start recv") - cmd = sever_run_cmd - #print(' '.join(cmd)) - shell_recv.send(' '.join(cmd) + "\n") - # time.sleep(1000) - - def start_sender(): - output = " " - #print(get_datetime(), "start sender") - - cmd = client_run_cmd - #print(get_datetime(), ' '.join(cmd)) - shell_sender.send(' '.join(cmd) + "\n") - times=0 - doc = open('bwlog.txt', 'a+') - doc.seek(0) - doc.truncate() - doc.close() - while times < 15: - time.sleep(1) - times += 1 - if times <2: - if shell_sender.recv_ready(): - outbuf = shell_sender.recv(65535) - continue - if shell_sender.recv_ready(): - outbuf = shell_sender.recv(65535) - if len(outbuf) == 0: - break - output += outbuf.decode("utf-8", "ignore") - doc = open('bwlog.txt', 'a+') - doc.write(outbuf.decode("utf-8", "ignore")) - doc.close() - print(output) - start_recv() - time.sleep(1) - print("signal signal") - start_sender() - netctr = get_ssh("netctr") - print("start scp") - scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) - scp_client.put("bwlog.txt", "%s/." % (recv_wd)) - scp_client.close() - - - except Exception as e: - print(get_datetime(), "run_measure", e) - - finally: - print("measure finish") - -if __name__ == "__main__": - BWmeasure(1) From c8fd2393fde5dd0a8f31a2fcae50af5a7331e721 Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Tue, 27 Dec 2022 11:27:42 +0800 Subject: [PATCH 05/12] Delete clientbwlog.txt --- bwmur/tests/clientbwlog.txt | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 bwmur/tests/clientbwlog.txt diff --git a/bwmur/tests/clientbwlog.txt b/bwmur/tests/clientbwlog.txt deleted file mode 100644 index 46e8df3..0000000 --- a/bwmur/tests/clientbwlog.txt +++ /dev/null @@ -1,19 +0,0 @@ -Connecting to host 127.0.0.1, port 8000 -[ 5] local 127.0.0.1 port 39098 connected to 127.0.0.1 port 8000 -[ ID] Interval Transfer Bitrate Retr Cwnd -[ 5] 0.00-1.00 sec 3.28 GBytes 28.1 Gbits/sec 0 3.12 MBytes -[ 5] 1.00-2.00 sec 4.28 GBytes 36.7 Gbits/sec 0 3.12 MBytes -[ 5] 2.00-3.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes -[ 5] 3.00-4.00 sec 4.29 GBytes 36.8 Gbits/sec 0 3.12 MBytes -[ 5] 4.00-5.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes -[ 5] 5.00-6.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes -[ 5] 6.00-7.00 sec 4.30 GBytes 36.9 Gbits/sec 0 3.12 MBytes -[ 5] 7.00-8.00 sec 4.26 GBytes 36.6 Gbits/sec 0 3.12 MBytes -[ 5] 8.00-9.00 sec 4.27 GBytes 36.7 Gbits/sec 0 3.12 MBytes -[ 5] 9.00-10.00 sec 4.28 GBytes 36.8 Gbits/sec 0 3.12 MBytes -- - - - - - - - - - - - - - - - - - - - - - - - - -[ ID] Interval Transfer Bitrate Retr -[ 5] 0.00-10.00 sec 41.8 GBytes 35.9 Gbits/sec 0 sender -[ 5] 0.00-10.04 sec 41.8 GBytes 35.7 Gbits/sec receiver - -iperf Done. From 026dba506a5ea2a56976ab0fb4839d1ba539d047 Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Tue, 27 Dec 2022 11:27:53 +0800 Subject: [PATCH 06/12] Delete .test_bwmeasure.py.swp --- bwmur/tests/.test_bwmeasure.py.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 bwmur/tests/.test_bwmeasure.py.swp diff --git a/bwmur/tests/.test_bwmeasure.py.swp b/bwmur/tests/.test_bwmeasure.py.swp deleted file mode 100644 index 26a796805bd7bca5c80d997e16fc8c6c061ce2c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&u`N(6vsUTC$xd!z+wFGvP*-Sv|0I4P4t4;&T?>*bkpPs6`r2FdN8SRG=fp(RUUtdnr zubUUijZfy77so}>iu#-#+&P=7b8&$gF7Mc1TFI{0dJ9QUy zAOR$BW&)MXgZ{>-w-a^U+HG%(UcWXv^ES8u2_OL^fCP{L5%Yvd!O zf1$a;jiz7wt8b>*Apsg|Kk7u zk1K?H2Ymy51|5SwfZl_ig7!gsp!=Xr&}Goc1|bUc8uSQs2Xq^B3v?5-4sz%GbZ22h z0!RP}AOR$R1dsp{Kmter2_S+0O@Km6$2@%-Co98g%O@GnsVm2ZLhP2m>+)~`%owgM z^&ShWndKf8l4i`Z+UW*#K+__NH8<77hQ?;9vbD3+LfY0JpI=*Hp28AOlw9XM8l*JZ zc^G!#iD+PH)b$o&&6W^ss-)B^<;H}%l5R`Wo?l`x)D>^j`Kg1IxWj|Pz1q&TUbVj* zK6x=3wkemX%7i=`cu*SK_vX1RGd9ay&1$ZO29Z~DY6v!<(n`mf&=9IjSTZ{fVJ2C@ zTZ=hMjI+1mI2RKh$G#7EbFB&>UX;#8MNujM{T!BMLoh1M^lTqIr12=Y^c$mOo0c>8 tJHhpqT-wx%f9e@+3f%T From 176eb45a8e7d2817b5c6e0d74c7c14d481aa1a9a Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Tue, 27 Dec 2022 11:28:02 +0800 Subject: [PATCH 07/12] Delete severbwlog.txt --- bwmur/tests/severbwlog.txt | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 bwmur/tests/severbwlog.txt diff --git a/bwmur/tests/severbwlog.txt b/bwmur/tests/severbwlog.txt deleted file mode 100644 index d4d2a99..0000000 --- a/bwmur/tests/severbwlog.txt +++ /dev/null @@ -1,20 +0,0 @@ ------------------------------------------------------------ -Server listening on 8000 ------------------------------------------------------------ -Accepted connection from 127.0.0.1, port 39090 -[ 5] local 127.0.0.1 port 8000 connected to 127.0.0.1 port 39098 -[ ID] Interval Transfer Bitrate -[ 5] 0.00-1.00 sec 3.09 GBytes 26.5 Gbits/sec -[ 5] 1.00-2.00 sec 4.28 GBytes 36.8 Gbits/sec -[ 5] 2.00-3.00 sec 4.27 GBytes 36.6 Gbits/sec -[ 5] 3.00-4.00 sec 4.28 GBytes 36.8 Gbits/sec -[ 5] 4.00-5.00 sec 4.27 GBytes 36.7 Gbits/sec -[ 5] 5.00-6.00 sec 4.27 GBytes 36.7 Gbits/sec -[ 5] 6.00-7.00 sec 4.30 GBytes 36.9 Gbits/sec -[ 5] 7.00-8.00 sec 4.26 GBytes 36.6 Gbits/sec -[ 5] 8.00-9.00 sec 4.27 GBytes 36.7 Gbits/sec -[ 5] 9.00-10.00 sec 4.28 GBytes 36.8 Gbits/sec -[ 5] 10.00-10.04 sec 190 MBytes 36.7 Gbits/sec -- - - - - - - - - - - - - - - - - - - - - - - - - -[ ID] Interval Transfer Bitrate -[ 5] 0.00-10.04 sec 41.8 GBytes 35.7 Gbits/sec receiver From 7bfcce16d152d161b622b29db3687b34fc98d54a Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Tue, 27 Dec 2022 11:28:39 +0800 Subject: [PATCH 08/12] Update machines.json --- bwmur/tests/machines.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bwmur/tests/machines.json b/bwmur/tests/machines.json index 5c9c30d..bdd1b7f 100644 --- a/bwmur/tests/machines.json +++ b/bwmur/tests/machines.json @@ -3,13 +3,11 @@ "recv_1" : { "host" : "127.0.0.1", "bw_port" : "8000", - "user" : "wang", - "pwd" : "123456789wang@" }, "netctr" : { "host" : "127.0.0.1", "ssh_port" : "22", - "user" : "root", + "user" : "", "pwd" : "" } From 3b7205915303557a2489de53f48df47e1aca2ef4 Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Sun, 29 Jan 2023 14:49:11 +0800 Subject: [PATCH 09/12] Update bwmurclient.py --- bwmur/bwmurclient.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bwmur/bwmurclient.py b/bwmur/bwmurclient.py index 058af48..3e71e4e 100644 --- a/bwmur/bwmurclient.py +++ b/bwmur/bwmurclient.py @@ -35,10 +35,6 @@ def get_bw(name): machines = json.loads(f.read()) if name not in machines: raise ValueError("Not find such mahcine") - - #print("iperf") - #print(machines[name]["host"]) - #print(machines[name]["bw_port"]) sever_ip = machines[name]["host"] sever_port = machines[name]["bw_port"] print(machines[name]["host"]) @@ -61,7 +57,6 @@ def get_ssh(name): return client def BWmeasure(matches_num): - #print(get_datetime(), "start measure") try: output =" " get_bw("recv_%d" % (matches_num)) From d5289a69123937f9f0a07f19303521e5808d6b3b Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Sun, 29 Jan 2023 14:57:06 +0800 Subject: [PATCH 10/12] Update bwmursever.py --- bwmur/bwmursever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bwmur/bwmursever.py b/bwmur/bwmursever.py index 205e7b4..8adff43 100644 --- a/bwmur/bwmursever.py +++ b/bwmur/bwmursever.py @@ -12,7 +12,7 @@ socket_timeout_sec = 240 machines_file = "machines.json" sever_port = "8000" -sever_ip = "20.81.187.38" +sever_ip = "0.0.0.0" recv_wd = "/home/wang/testnet/" sever_run_cmd = [ From 952cbaf6403f9027c050b84639235769b9db05e2 Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Sun, 29 Jan 2023 15:03:33 +0800 Subject: [PATCH 11/12] Update bwmursever.py --- bwmur/bwmursever.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bwmur/bwmursever.py b/bwmur/bwmursever.py index 8adff43..3a05d13 100644 --- a/bwmur/bwmursever.py +++ b/bwmur/bwmursever.py @@ -75,10 +75,13 @@ def BWmeasure(matches_num): doc.write(status) doc.close() netctr = get_ssh("netctr") + ''' + At present, the method of uploading files through SCP is not adopted. print("start scp") scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) scp_client.put("severbwlog.txt", "%s/." % (recv_wd)) scp_client.close() + ''' except Exception as e: From 60c9c37688124297f70d91f7a4f2d99368a0b32d Mon Sep 17 00:00:00 2001 From: wangyf2001 <64457943+wangyf2001@users.noreply.github.com> Date: Sun, 29 Jan 2023 15:04:27 +0800 Subject: [PATCH 12/12] Update machines.json --- machines.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines.json b/machines.json index 5c9c30d..b72e6fb 100644 --- a/machines.json +++ b/machines.json @@ -3,8 +3,8 @@ "recv_1" : { "host" : "127.0.0.1", "bw_port" : "8000", - "user" : "wang", - "pwd" : "123456789wang@" + "user" : "Your user name", + "pwd" : "Your password" }, "netctr" : { "host" : "127.0.0.1",