-
Notifications
You must be signed in to change notification settings - Fork 1
/
01-CIME-fix-download-server-fallback.patch
79 lines (73 loc) · 4.33 KB
/
01-CIME-fix-download-server-fallback.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
check_input_data() only considers the download attempt of the last missing file
to fallback to other servers. This patch fixes the fallback mechanism to jump
to the following server in the list if any of the missing files fails to download.
author: Alex Domingo (Vrije Universiteit Brussel)
diff -W 126 -Nru cesm-2.1.3-orig/cime/scripts/lib/CIME/case/check_input_data.py cesm-2.1.3/cime/scripts/lib/CIME/case/check_input_data.py
--- a/cime/scripts/lib/CIME/case/check_input_data.py 2020-08-14 22:11:33.487882000 +0200
+++ b/cime/scripts/lib/CIME/case/check_input_data.py 2020-08-15 00:06:17.352015069 +0200
@@ -276,7 +276,8 @@
data_list_files = find_files(data_list_dir, "*.input_data_list")
expect(data_list_files, "No .input_data_list files found in dir '{}'".format(data_list_dir))
- no_files_missing = True
+ chksum_required = ["gftp", "ftp", "wget"]
+
if download:
if protocol not in vars(CIME.Servers):
logger.warning("Client protocol {} not enabled".format(protocol))
@@ -295,6 +296,7 @@
if not server:
return False
+ installation_status = True
for data_list_file in data_list_files:
logging.info("Loading input file list: '{}'".format(data_list_file))
with open(data_list_file, "r") as fd:
@@ -308,6 +310,7 @@
if description.endswith('datapath'):
continue
if(full_path):
+ file_installed = False
# expand xml variables
full_path = case.get_resolved_value(full_path)
rel_path = full_path.replace(input_data_root, "")
@@ -320,9 +323,9 @@
if not os.path.exists(full_path):
logging.warning("Model {} missing file {} = '{}'".format(model, description, full_path))
if download:
- logging.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
- no_files_missing = False
+ logging.warning("Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
else:
+ file_installed = True
logging.debug(" Found input file: '{}'".format(full_path))
else:
# There are some special values of rel_path that
@@ -335,24 +338,28 @@
if ("/" in rel_path and not os.path.exists(full_path)):
logger.warning(" Model {} missing file {} = '{}'".format(model, description, full_path))
- no_files_missing = False
if (download):
- no_files_missing = _download_if_in_repo(server,
+ file_installed = _download_if_in_repo(server,
input_data_root, rel_path.strip(os.sep),
isdirectory=isdirectory)
- if no_files_missing:
+ if file_installed and protocol in chksum_required:
verify_chksum(input_data_root, rundir, rel_path.strip(os.sep), isdirectory)
else:
- if chksum:
+ if chksum and protocol in chksum_required:
verify_chksum(input_data_root, rundir, rel_path.strip(os.sep), isdirectory)
logger.info("Chksum passed for file {}".format(os.path.join(input_data_root,rel_path)))
+ file_installed = True
logging.debug(" Already had input file: '{}'".format(full_path))
+
+ if not file_installed:
+ installation_status = False
+
else:
model = os.path.basename(data_list_file).split('.')[0]
logging.warning("Model {} no file specified for {}".format(model, description))
- return no_files_missing
+ return installation_status
def verify_chksum(input_data_root, rundir, filename, isdirectory):
"""