diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..894a44c --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..665441f --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +docker_image := net-env +docker_file := dockers/Dockerfile + +all: net-env + +net-env: + docker pull ubuntu:20.04 + docker build . --build-arg UID=$(shell id -u) --build-arg GUID=$(shell id -g) -f $(docker_file) -t ${docker_image} + diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..75bbc3d --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,41 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: + branches: + include: + - "*" + exclude: + - upstream/* + + +pool: + vmImage: 'ubuntu-latest' + +steps: + +- checkout: self + +- script: make + displayName: 'build network measure environment' + +- script: cd /home ; sudo wget https://github.com/OpenNetLab/OpenNetLab-P2P-Measurment/blob/v-wangyif/std-process-pitch/machines.json + displayName: 'make network measure environment' + +- script: docker run -v /home:/app -w /app --net=host --name net_evals net-env:latest python3 /home/onl/bwmur/bwmursever.py && docker run -v /home:/app -w /app --net=host --name net_evalc net-env:latest python3 /home/onl/bwmur/bwmurclient.py + displayName: 'Run test' + + +- script: docker image tag net-env:latest opennetlab.azurecr.io/net-env:latest + displayName: 'Tag network environment image' + +- task: Docker@2 + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + inputs: + containerRegistry: 'opennetlab Azure registry' + repository: 'net-env' + command: 'push' + tags: 'latest' + displayName: "Push net-env image" diff --git a/bwmur/bwmurclient.py b/bwmur/bwmurclient.py new file mode 100644 index 0000000..3e71e4e --- /dev/null +++ b/bwmur/bwmurclient.py @@ -0,0 +1,86 @@ +import paramiko +import time +import random +import json +import os +import sys +from scp import SCPClient +import numpy as np +import copy +import time + +socket_timeout_sec = 240 +machines_file = "machines.json" +sever_port = "8000" +sever_ip = "127.0.0.1" +recv_wd = "/home" + +sever_run_cmd = [ + "iperf3 -s -p %s -i 1 -1" % ( + sever_port)] +client_run_cmd = [ + "iperf3 -c %s -p %s" % ( + sever_ip, sever_port)] + +def get_datetime(): + now = int(round(time.time() * 1000)) + ret = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now / 1000)) + return ret + + +def get_bw(name): + global sever_ip + global sever_port + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + sever_ip = machines[name]["host"] + sever_port = machines[name]["bw_port"] + print(machines[name]["host"]) + print(sever_ip) + + + +def get_ssh(name): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + client.connect(hostname=machines[name]["host"], port=machines[name]["ssh_port"], username=machines[name]["user"], + password=machines[name]["pwd"]) + + + return client + +def BWmeasure(matches_num): + try: + output =" " + get_bw("recv_%d" % (matches_num)) + client_run_cmd = ["iperf3 -c %s -p %s" % (sever_ip, sever_port)] + status = os.popen(" ".join(client_run_cmd)).read() + print(" ".join(client_run_cmd)) + cmd = client_run_cmd + doc = open('clientbwlog.txt', 'a+') + doc.seek(0) + doc.truncate() + doc.write(status) + doc.close() + netctr = get_ssh("netctr") + print("start scp") + scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) + scp_client.put("clientbwlog.txt", "%s/." % (recv_wd)) + scp_client.close() + + + except Exception as e: + print(get_datetime(), "run_measure", e) + + finally: + print("measure finish") + +if __name__ == "__main__": + BWmeasure(1) diff --git a/bwmur/bwmursever.py b/bwmur/bwmursever.py new file mode 100644 index 0000000..3a05d13 --- /dev/null +++ b/bwmur/bwmursever.py @@ -0,0 +1,94 @@ +import paramiko +import time +import random +import json +import os +import sys +from scp import SCPClient +import numpy as np +import copy +import time + +socket_timeout_sec = 240 +machines_file = "machines.json" +sever_port = "8000" +sever_ip = "0.0.0.0" +recv_wd = "/home/wang/testnet/" + +sever_run_cmd = [ + "iperf3 -s -p %s -i 1 -1" % ( + sever_port)] +client_run_cmd = [ + "iperf3 -c %s -p %s" % ( + sever_ip, sever_port)] + +def get_datetime(): + now = int(round(time.time() * 1000)) + ret = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now / 1000)) + return ret + + +def get_bw(name): + global sever_ip + global sever_port + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + #print("iperf") + #print(machines[name]["host"]) + #print(machines[name]["bw_port"]) + sever_ip = machines[name]["host"] + sever_port = machines[name]["bw_port"] + + + +def get_ssh(name): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + with open(machines_file, 'r') as f: + machines = json.loads(f.read()) + if name not in machines: + raise ValueError("Not find such mahcine") + + client.connect(hostname=machines[name]["host"], port=machines[name]["ssh_port"], username=machines[name]["user"], + password=machines[name]["pwd"]) + + #print(machines[name]["host"]) + #print(machines[name]["ssh_port"]) + #print(machines[name]["user"]) + + return client + +def BWmeasure(matches_num): + #print(get_datetime(), "start measure") + try: + output =" " + get_bw("recv_%d" % (matches_num)) + sever_run_cmd = ["iperf3 -s -p %s -i 1 -1" % (sever_port)] + status = os.popen(" ".join(sever_run_cmd)).read() + cmd = sever_run_cmd + doc = open('severbwlog.txt', 'a+') + doc.seek(0) + doc.truncate() + doc.write(status) + doc.close() + netctr = get_ssh("netctr") + ''' + At present, the method of uploading files through SCP is not adopted. + print("start scp") + scp_client = SCPClient(netctr.get_transport(), socket_timeout=30.0) + scp_client.put("severbwlog.txt", "%s/." % (recv_wd)) + scp_client.close() + ''' + + + except Exception as e: + print(get_datetime(), "run_measure", e) + + finally: + print("measure finish") + +if __name__ == "__main__": + BWmeasure(1) diff --git a/bwmur/tests/machines.json b/bwmur/tests/machines.json new file mode 100644 index 0000000..bdd1b7f --- /dev/null +++ b/bwmur/tests/machines.json @@ -0,0 +1,14 @@ +{ + + "recv_1" : { + "host" : "127.0.0.1", + "bw_port" : "8000", + }, + "netctr" : { + "host" : "127.0.0.1", + "ssh_port" : "22", + "user" : "", + "pwd" : "" + } + +} diff --git a/bwmur/tests/test_bwmeasure.py b/bwmur/tests/test_bwmeasure.py new file mode 100644 index 0000000..ab01221 --- /dev/null +++ b/bwmur/tests/test_bwmeasure.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os, json, pytest, subprocess, time +from tempfile import NamedTemporaryFile + + +cur_dir = os.path.dirname(os.path.abspath(__file__)) + + +def check_bws(): + file_path = cur_dir + "/../bwmursever.py" + cmds = ["python3",file_path] + cmds = (" ".join(cmds)) + subprocess.Popen(cmds,shell = True) + #os.system(cmds) + + +def check_bwc(): + file_path = cur_dir + "/../bwmurclient.py" + cmdc = ["python3",file_path] + cmdc = (" ".join(cmdc)) + #cmd_result = subprocess.Popen(cmdc,shell = True).stdout.read() + cmd_result = os.popen(cmdc).read() + print(cmd_result) + assert "scp" in cmd_result + + + +def test_bw(): + check_bws() + print("OK") + check_bwc() diff --git a/dockers/Dockerfile b/dockers/Dockerfile new file mode 100644 index 0000000..51c2346 --- /dev/null +++ b/dockers/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:20.04 + +# Add ONL user +ARG USER=onl + +WORKDIR /home/${USER} + +ARG DEBIAN_FRONTEND=noninteractive +# Install dependency +RUN apt-get update && apt-get install -y \ + python3-pip wget unzip gocr imagemagick iperf3 + +RUN pip3 install pytest numpy requests soundfile paramiko scp + +COPY bwmur bwmur diff --git a/machines.json b/machines.json new file mode 100644 index 0000000..b72e6fb --- /dev/null +++ b/machines.json @@ -0,0 +1,16 @@ +{ + + "recv_1" : { + "host" : "127.0.0.1", + "bw_port" : "8000", + "user" : "Your user name", + "pwd" : "Your password" + }, + "netctr" : { + "host" : "127.0.0.1", + "ssh_port" : "22", + "user" : "root", + "pwd" : "" + } + +}