From a7d4a4cbb8e9e07885ad51fdb4dffdabc375c0ab Mon Sep 17 00:00:00 2001 From: xschlef Date: Mon, 9 Apr 2018 15:40:09 +0200 Subject: [PATCH 1/8] added vars plugin that extends metadata with arbitrary text variable for additional meta information --- src/lib/Bcfg2/Server/Plugins/Vars.py | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Plugins/Vars.py diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py new file mode 100644 index 0000000000..0e2c9eeed5 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -0,0 +1,88 @@ +""" Support for client ACLs based on IP address and client metadata """ + +import os +import sys +import Bcfg2.Server.Plugin +import lxml +import copy +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Server.Cache import Cache + + +class VarsFile(Bcfg2.Server.Plugin.StructFile): + """ representation of Vars vars.xml """ + def __init__(self, name, core, should_monitor=False): + """ + :param name: The filename of this vars file. + :type name: string + :param core: The Bcfg2.Server.Core initializing the Vars plugin + :type core: Bcfg2.Server.Core + """ + Bcfg2.Server.Plugin.StructFile.__init__(self, name, + should_monitor=should_monitor) + self.name = name + self.core = core + self.cache = Cache("Vars") + + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + self.cache.clear() + + def get_vars(self, metadata): + if metadata.hostname in self.cache: + self.debug_log("Vars: Found cached vars for %s." % metadata.hostname) + return copy.copy(self.cache[metadata.hostname]) + rv = dict() + for el in self.Match(metadata): + # only evaluate var tags, this is extensible in the future + if el.tag == "var": + self.debug_log(el) + if 'name' not in el.attrib: + # if we have a correct schema, this should not happen + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Vars: Invalid structure of vars.xml. Missing name attribute for variable.") + rv[el.get('name')] = el.text + self.cache[metadata.hostname] = copy.copy(rv) + return rv + + def validate_data(self): + """ ensure that the data in this object validates against the + XML schema for the vars file (if a schema exists) """ + schemafile = self.name.replace(".xml", ".xsd") + if os.path.exists(schemafile): + try: + schema = lxml.etree.XMLSchema(file=schemafile) + except lxml.etree.XMLSchemaParseError: + err = sys.exc_info()[1] + raise PluginExecutionError("Failed to process schema for %s: " + "%s" % (self.name, err)) + else: + # no schema exists + return True + + if not schema.validate(self.xdata): + raise PluginExecutionError("Data for %s fails to validate; run " + "bcfg2-lint for more details" % + self.name) + +class Vars(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ add additional info to the metadata object based on entries in the vars.xml """ + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.Connector.__init__(self) + self.vars_file = VarsFile(os.path.join(self.data, 'vars.xml'), + core, + should_monitor=True) + + def get_additional_data(self, metadata): + """ """ + self.debug_log("Vars: Getting vars for %s" % metadata.hostname) + return self.vars_file.get_vars(metadata) + + def set_debug(self, debug): + rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) + self.vars_file.set_debug(debug) + return rv + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ From 4a4acb706e4af773f964ffba0fbb5022e38cf576 Mon Sep 17 00:00:00 2001 From: xschlef Date: Mon, 9 Apr 2018 15:58:56 +0200 Subject: [PATCH 2/8] fix coding style and docstrings --- src/lib/Bcfg2/Server/Plugins/Vars.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index 0e2c9eeed5..e6389df04b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -1,34 +1,35 @@ """ Support for client ACLs based on IP address and client metadata """ +import copy import os import sys -import Bcfg2.Server.Plugin + import lxml -import copy -from Bcfg2.Server.Plugin import PluginExecutionError + +import Bcfg2.Server.Plugin from Bcfg2.Server.Cache import Cache +from Bcfg2.Server.Plugin import PluginExecutionError class VarsFile(Bcfg2.Server.Plugin.StructFile): """ representation of Vars vars.xml """ def __init__(self, name, core, should_monitor=False): - """ - :param name: The filename of this vars file. - :type name: string - :param core: The Bcfg2.Server.Core initializing the Vars plugin - :type core: Bcfg2.Server.Core - """ Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) self.name = name self.core = core self.cache = Cache("Vars") + __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ + def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.cache.clear() + Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + def get_vars(self, metadata): + """ gets all var tags from the vars.xml """ if metadata.hostname in self.cache: self.debug_log("Vars: Found cached vars for %s." % metadata.hostname) return copy.copy(self.cache[metadata.hostname]) @@ -65,8 +66,9 @@ def validate_data(self): "bcfg2-lint for more details" % self.name) + class Vars(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): + Bcfg2.Server.Plugin.Connector): """ add additional info to the metadata object based on entries in the vars.xml """ def __init__(self, core): @@ -76,11 +78,15 @@ def __init__(self, core): core, should_monitor=True) + __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ + def get_additional_data(self, metadata): - """ """ self.debug_log("Vars: Getting vars for %s" % metadata.hostname) return self.vars_file.get_vars(metadata) + get_additional_data.__doc__ = \ + Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ + def set_debug(self, debug): rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) self.vars_file.set_debug(debug) From f9d1133779e2fb0afce020b9e12cbb11f7f1ec09 Mon Sep 17 00:00:00 2001 From: xschlef Date: Mon, 9 Apr 2018 16:05:53 +0200 Subject: [PATCH 3/8] fixed copy and paste error --- src/lib/Bcfg2/Server/Plugins/Vars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index e6389df04b..ad5c2d9a52 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -1,4 +1,4 @@ -""" Support for client ACLs based on IP address and client metadata """ +""" Support for metadata.Vars that can contain arbitrary strings for faster generation of templates """ import copy import os From f5626ef8369f9b2ad28afff1998a3dc069b8ea02 Mon Sep 17 00:00:00 2001 From: xschlef Date: Mon, 16 Apr 2018 10:08:05 +0200 Subject: [PATCH 4/8] modified docstring --- src/lib/Bcfg2/Server/Plugins/Vars.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index ad5c2d9a52..c68ad8ea8b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -1,4 +1,7 @@ -""" Support for metadata.Vars that can contain arbitrary strings for faster generation of templates """ +""" + Support for metadata.Vars + these contain arbitrary strings for faster generation of templates +""" import copy import os From fd2843f456748ea3b2dd32507f0cea2042fcaebe Mon Sep 17 00:00:00 2001 From: xschlef Date: Thu, 26 Apr 2018 15:14:50 +0200 Subject: [PATCH 5/8] add support for a type attribute, so json encoded strings can be added as vars. Also clear the metadata cache if we change the vars.xml --- src/lib/Bcfg2/Server/Plugins/Vars.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index c68ad8ea8b..b9adf63396 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -13,14 +13,24 @@ from Bcfg2.Server.Cache import Cache from Bcfg2.Server.Plugin import PluginExecutionError +has_json = False +try: + import json + + has_json = True +except ImportError: + pass + class VarsFile(Bcfg2.Server.Plugin.StructFile): """ representation of Vars vars.xml """ + def __init__(self, name, core, should_monitor=False): Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) self.name = name self.core = core + # even though we are a connector plugin, keep a local cache self.cache = Cache("Vars") __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ @@ -28,6 +38,7 @@ def __init__(self, name, core, should_monitor=False): def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.cache.clear() + self.core.metadata_cache.expire() Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ @@ -45,10 +56,15 @@ def get_vars(self, metadata): # if we have a correct schema, this should not happen raise Bcfg2.Server.Plugin.PluginExecutionError( "Vars: Invalid structure of vars.xml. Missing name attribute for variable.") - rv[el.get('name')] = el.text + if has_json and el.get('type') == "json": + rv[el.get('name')] = json.loads(el.text) + else: + rv[el.get('name')] = el.text + self.cache[metadata.hostname] = copy.copy(rv) return rv + def validate_data(self): """ ensure that the data in this object validates against the XML schema for the vars file (if a schema exists) """ @@ -94,4 +110,5 @@ def set_debug(self, debug): rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) self.vars_file.set_debug(debug) return rv + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ From 8ae81ed9f52acc4fe2a0fd519a7fdad30f6e678c Mon Sep 17 00:00:00 2001 From: xschlef Date: Thu, 26 Apr 2018 15:47:13 +0200 Subject: [PATCH 6/8] further coding style fixes --- src/lib/Bcfg2/Server/Plugins/Vars.py | 32 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index b9adf63396..9ea439ddcb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -13,17 +13,16 @@ from Bcfg2.Server.Cache import Cache from Bcfg2.Server.Plugin import PluginExecutionError -has_json = False +HAS_JSON = False try: import json - - has_json = True + HAS_JSON = True except ImportError: pass class VarsFile(Bcfg2.Server.Plugin.StructFile): - """ representation of Vars vars.xml """ + """ representation of Vars vars.xml. Manages caching and handles file events. """ def __init__(self, name, core, should_monitor=False): Bcfg2.Server.Plugin.StructFile.__init__(self, name, @@ -45,7 +44,8 @@ def Index(self): def get_vars(self, metadata): """ gets all var tags from the vars.xml """ if metadata.hostname in self.cache: - self.debug_log("Vars: Found cached vars for %s." % metadata.hostname) + self.debug_log("Vars: Found cached vars for %s." % + metadata.hostname) return copy.copy(self.cache[metadata.hostname]) rv = dict() for el in self.Match(metadata): @@ -55,8 +55,9 @@ def get_vars(self, metadata): if 'name' not in el.attrib: # if we have a correct schema, this should not happen raise Bcfg2.Server.Plugin.PluginExecutionError( - "Vars: Invalid structure of vars.xml. Missing name attribute for variable.") - if has_json and el.get('type') == "json": + "Vars: Invalid structure of vars.xml. " + "Missing name attribute for variable.") + if HAS_JSON and el.get('type') == "json": rv[el.get('name')] = json.loads(el.text) else: rv[el.get('name')] = el.text @@ -64,7 +65,6 @@ def get_vars(self, metadata): self.cache[metadata.hostname] = copy.copy(rv) return rv - def validate_data(self): """ ensure that the data in this object validates against the XML schema for the vars file (if a schema exists) """ @@ -74,21 +74,25 @@ def validate_data(self): schema = lxml.etree.XMLSchema(file=schemafile) except lxml.etree.XMLSchemaParseError: err = sys.exc_info()[1] - raise PluginExecutionError("Failed to process schema for %s: " - "%s" % (self.name, err)) + raise PluginExecutionError( + "Failed to process schema for %s: " + "%s" % (self.name, err)) else: # no schema exists return True if not schema.validate(self.xdata): - raise PluginExecutionError("Data for %s fails to validate; run " - "bcfg2-lint for more details" % - self.name) + raise PluginExecutionError( + "Data for %s fails to validate; run " + "bcfg2-lint for more details" % self.name) class Vars(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): - """ add additional info to the metadata object based on entries in the vars.xml """ + """ The vars plugins adds additional info to the metadata object + based on entries in the vars.xml. Data can even be serialized + with json if desired. + """ def __init__(self, core): Bcfg2.Server.Plugin.Plugin.__init__(self, core) From da6d066f231933d0f73a26a6f07ab7181b7dbc85 Mon Sep 17 00:00:00 2001 From: xschlef Date: Thu, 26 Apr 2018 16:16:32 +0200 Subject: [PATCH 7/8] again a line too long --- src/lib/Bcfg2/Server/Plugins/Vars.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index 9ea439ddcb..4d66ce848b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -22,7 +22,8 @@ class VarsFile(Bcfg2.Server.Plugin.StructFile): - """ representation of Vars vars.xml. Manages caching and handles file events. """ + """ representation of Vars vars.xml. + Manages caching and handles file events. """ def __init__(self, name, core, should_monitor=False): Bcfg2.Server.Plugin.StructFile.__init__(self, name, From 598cfb71887f8ed30c9f70b0c54322a9f000d0db Mon Sep 17 00:00:00 2001 From: xschlef Date: Tue, 3 Jul 2018 10:29:28 +0200 Subject: [PATCH 8/8] fixed vars plugin for xincludes --- src/lib/Bcfg2/Server/Plugins/Vars.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Vars.py b/src/lib/Bcfg2/Server/Plugins/Vars.py index 4d66ce848b..a8e8da1976 100644 --- a/src/lib/Bcfg2/Server/Plugins/Vars.py +++ b/src/lib/Bcfg2/Server/Plugins/Vars.py @@ -49,21 +49,18 @@ def get_vars(self, metadata): metadata.hostname) return copy.copy(self.cache[metadata.hostname]) rv = dict() - for el in self.Match(metadata): - # only evaluate var tags, this is extensible in the future - if el.tag == "var": - self.debug_log(el) - if 'name' not in el.attrib: - # if we have a correct schema, this should not happen - raise Bcfg2.Server.Plugin.PluginExecutionError( - "Vars: Invalid structure of vars.xml. " - "Missing name attribute for variable.") - if HAS_JSON and el.get('type') == "json": - rv[el.get('name')] = json.loads(el.text) - else: - rv[el.get('name')] = el.text - + for el in self.XMLMatch(metadata).xpath("//var"): + if 'name' not in el.attrib: + # if we have a correct schema, this should not happen + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Vars: Invalid structure of vars.xml. " + "Missing name attribute for variable.") + if HAS_JSON and el.get('type') == "json": + rv[el.get('name')] = json.loads(el.text) + else: + rv[el.get('name')] = el.text self.cache[metadata.hostname] = copy.copy(rv) + return rv def validate_data(self):