Skip to content

Commit

Permalink
Apply style changes using Ruff
Browse files Browse the repository at this point in the history
  • Loading branch information
ma595 committed Oct 18, 2024
1 parent d38b46e commit a98c3e4
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 44 deletions.
112 changes: 70 additions & 42 deletions tests/python/integration/test_partitioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,37 @@

_log = logging.getLogger(__name__)


@pytest.fixture(scope="module")
def get_data():
"""
Obtain sample test data using a fixture
Function currently unused.
"""
from zipfile import ZipFile
urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip')
urllib.urlopen('ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip')


urllib.urlopen(
"ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_output.zip"
)
urllib.urlopen(
"ftp://ftp.fluxdata.org/.ameriflux_downloads/.test/US-ARc_sample_input.zip"
)

input_zip = "US-ARc_sample_input.zip"
output_zip = "US-ARc_sample_output.zip"

with ZipFile(input_zip) as zi, ZipFile(output_zip) as zo:
zi.extractall(path='tests/data/test_input')
zo.extractall(path='tests/data/test_reference')
zi.extractall(path="tests/data/test_input")
zo.extractall(path="tests/data/test_reference")


def equal_csv(csv_1, csv_2):
"""
Check equality of two csv files.
"""
_log.info("Check csv equality")
start = time.time()
with open(csv_1, 'r') as t1, open(csv_2, 'r') as t2:
with open(csv_1, "r") as t1, open(csv_2, "r") as t2:
fileone = t1.readlines()
filetwo = t2.readlines()
for line in filetwo:
Expand All @@ -47,33 +54,37 @@ def equal_csv(csv_1, csv_2):
@pytest.fixture
def setup_data():
"""
Set up input data for run_partition_nt test.
Create data directory for tests './tests/integration/step10' and copy
Set up input data for run_partition_nt test.
Create data directory for tests './tests/integration/step10' and copy
data from expected output ('./datadir/test_output/US-ARc_sample_output')
to this directory.
"""
try:
os.mkdir('tests/integration/data/step_10')
os.mkdir('tests/data/test_reference')
os.mkdir("tests/integration/data/step_10")
os.mkdir("tests/data/test_reference")
except OSError as e:
if e.errno == errno.EEXIST:
print("directory exists")

testdata = 'tests/python/integration/input/step_10/US-ARc_sample_input'

copy_tree('tests/data/test_input/', testdata)

refoutdir = 'tests/data/test_reference/US-ARc_sample_output'

copy_tree(os.path.join(refoutdir, '07_meteo_proc'), \
os.path.join(testdata, '07_meteo_proc'))
copy_tree(os.path.join(refoutdir, '08_nee_proc'), \
os.path.join(testdata, '08_nee_proc/'))
copy_tree(os.path.join(refoutdir, '02_qc_auto'), \
os.path.join(testdata, '02_qc_auto/'))



testdata = "tests/python/integration/input/step_10/US-ARc_sample_input"

copy_tree("tests/data/test_input/", testdata)

refoutdir = "tests/data/test_reference/US-ARc_sample_output"

copy_tree(
os.path.join(refoutdir, "07_meteo_proc"),
os.path.join(testdata, "07_meteo_proc"),
)
copy_tree(
os.path.join(refoutdir, "08_nee_proc"), os.path.join(testdata, "08_nee_proc/")
)
copy_tree(
os.path.join(refoutdir, "02_qc_auto"), os.path.join(testdata, "02_qc_auto/")
)


def test_run_partition_nt(setup_data):
"""
Run partition_nt on single percentile.
Expand All @@ -82,35 +93,52 @@ def test_run_partition_nt(setup_data):
refoutdir = "./tests/data/test_reference/"
siteid = "US-ARc"
sitedir = "US-ARc_sample_input"
years = [2005] # years = [2005, 2006]
years = [2005] # years = [2005, 2006]
# PROD_TO_COMPARE = ['c', 'y']
PROD_TO_COMPARE = ['y',]
PROD_TO_COMPARE = [
"y",
]
# PERC_TO_COMPARE = ['1.25', '3.75',]
PERC_TO_COMPARE = ['1.25',]

PERC_TO_COMPARE = [
"1.25",
]

from oneflux.tools.partition_nt import remove_previous_run, run_python
remove_previous_run(datadir=datadir, siteid=siteid, sitedir=sitedir, python=True,
prod_to_compare=PROD_TO_COMPARE, perc_to_compare=PERC_TO_COMPARE,
years_to_compare=years)

run_python(datadir=datadir, siteid=siteid, sitedir=sitedir, prod_to_compare=PROD_TO_COMPARE,
perc_to_compare=PERC_TO_COMPARE, years_to_compare=years)

remove_previous_run(
datadir=datadir,
siteid=siteid,
sitedir=sitedir,
python=True,
prod_to_compare=PROD_TO_COMPARE,
perc_to_compare=PERC_TO_COMPARE,
years_to_compare=years,
)

run_python(
datadir=datadir,
siteid=siteid,
sitedir=sitedir,
prod_to_compare=PROD_TO_COMPARE,
perc_to_compare=PERC_TO_COMPARE,
years_to_compare=years,
)

# check whether csv of output in input directory is same as csv of reference

# the generated output is actually in the "input" directory.
rootdir = os.path.join(datadir, sitedir, "10_nee_partition_nt")
nee_y_files = glob.glob(os.path.join(rootdir, "nee_y_1.25_US-ARc_2005*"))
nee_y_files = filter(lambda x: not x.endswith('_orig.csv'), nee_y_files)
nee_y_files = filter(lambda x: not x.endswith("_orig.csv"), nee_y_files)

# paths to the "reference" output data
refoutdir = os.path.join(refoutdir, "US-ARc_sample_output", "10_nee_partition_nt")
ref_nee_y_files = glob.glob(os.path.join(refoutdir, "nee_y_1.25_US-ARc_2005*"))

assert len(nee_y_files) == len(ref_nee_y_files)
for f, b in zip(nee_y_files, ref_nee_y_files):
print(f, b)
assert equal_csv(f, b)
assert equal_csv(f, b)

# clean up data.
# clean up data.
# shutil.rmtree(datadir)
5 changes: 3 additions & 2 deletions tests/python/test_context.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
'''
"""
For license information:
see LICENSE file or headers in oneflux.__init__.py
Expand All @@ -7,7 +7,8 @@
@author: Gilberto Pastorello
@contact: [email protected]
@date: 2017-01-31
'''
"""


def test_import_oneflux():
"""
Expand Down

0 comments on commit a98c3e4

Please sign in to comment.