diff --git a/.gitignore b/.gitignore index 75f3d20..3b318e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ *.pyc *.swp .idea +__pycache__ +dist +build +*.egg-info +*.stratolog diff --git a/Dockerfile.pycommonlog-build b/Dockerfile.pycommonlog-build new file mode 100644 index 0000000..763d394 --- /dev/null +++ b/Dockerfile.pycommonlog-build @@ -0,0 +1,8 @@ +FROM rackattack-nas.dc1:5000/ubuntu-dev-base:379308aa77bd0a19ebf12d868e7d19ee35a19d6c +RUN apt-get install -y python2.7 python2.7-dev +RUN curl 'https://bootstrap.pypa.io/pip/2.7/get-pip.py' | python2.7 +COPY requirements.txt /tmp/requirements.txt +COPY dev-requirements.txt /tmp/dev-requirements.txt +RUN python2.7 -m pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt +RUN python3 -m pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt +RUN rm /tmp/requirements.txt /tmp/dev-requirements.txt diff --git a/Makefile b/Makefile index ca2b7a5..cdaa18a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,29 @@ -all: test check_convention +.PHONY: all test build test_py2 test_py3 lint_py2 lint_py3 +ARTIFACT=dist/strato_common_log-*-py2.py3-none-any.whl +all: lint test build -test: - PYTHONPATH=$(PWD)/py python py/strato/common/log/tests/test.py +test: test_py2 test_py3 -check_convention: - pep8 py --max-line-length=109 +test_py3: + PYTHONPATH=$(PWD)/py python3 py/strato/common/log/tests/test.py + +test_py2: + PYTHONPATH=$(PWD)/py python2.7 py/strato/common/log/tests/test.py + +lint: lint_py2 lint_py3 + +lint_py3: + python3 -m pep8 py --max-line-length=120 + +lint_py2: + python2.7 -m pep8 py --max-line-length=120 + +build: $(ARTIFACT) + +clean: + find . -name *.pyc -delete + find . -name __pycache__ -delete + rm -rf dist */*.egg-info build *.stratolog + +$(ARTIFACT): + python3 -m build --wheel diff --git a/README.md b/README.md new file mode 100644 index 0000000..52bfc2e --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# pycommonlog + +# Build +```bash +skipper make -i all +``` + +# Demo + +```python +from strato.common.log import configurelogging +configurelogging.configureLogging('test-strato-log', forceDirectory=".") +import logging # noqa: E402 +logging.warning('Running test') +logging.error('Running test') +logging.progress('Running test') +logging.step('Running test') +logging.critical('Running test') +logging.success('Running test') +logging.debug('Running test') +try: + raise Exception('Test exception') +except Exception: + logging.exception('Running test') +``` \ No newline at end of file diff --git a/artifacts.yaml b/artifacts.yaml new file mode 100644 index 0000000..68e265d --- /dev/null +++ b/artifacts.yaml @@ -0,0 +1,2 @@ +pips: + - dist/strato_common_log-*-py2.py3-none-any.whl diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..dbe447e --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,3 @@ +build +pep8 +ipdb diff --git a/py/strato/__init__.py b/py/strato/__init__.py index 48e952a..e69de29 100644 --- a/py/strato/__init__.py +++ b/py/strato/__init__.py @@ -1,2 +0,0 @@ -import upseto.pythonnamespacejoin -__path__.extend(upseto.pythonnamespacejoin.join(globals())) diff --git a/py/strato/common/__init__.py b/py/strato/common/__init__.py index 48e952a..e69de29 100644 --- a/py/strato/common/__init__.py +++ b/py/strato/common/__init__.py @@ -1,2 +0,0 @@ -import upseto.pythonnamespacejoin -__path__.extend(upseto.pythonnamespacejoin.join(globals())) diff --git a/py/strato/common/log/config.py b/py/strato/common/log/config.py index bb4e18c..09379fe 100644 --- a/py/strato/common/log/config.py +++ b/py/strato/common/log/config.py @@ -39,4 +39,4 @@ "incremental": True } -exec os.environ.get("STRATO_CONFIG_LOGGING", "") +exec(os.environ.get("STRATO_CONFIG_LOGGING", "")) diff --git a/py/strato/common/log/configurelogging.py b/py/strato/common/log/configurelogging.py index 8109557..45e7d1a 100644 --- a/py/strato/common/log/configurelogging.py +++ b/py/strato/common/log/configurelogging.py @@ -7,12 +7,16 @@ import sys import atexit import json -import subprocess import signal +try: + import subprocess32 as subprocess +except ImportError: + import subprocess _name = None _registered_file_handles = dict() + def logFilename(name): return '%s/%s%s' % (config.LOGS_DIRECTORY, name, config.LOGS_SUFFIX) @@ -77,16 +81,21 @@ def changeHandlerLogLevelbyHandlerType(logger, logLevel, handlerType=None): [handler.setLevel(logLevel) for handler in logger.handlers if not handlerType or type(handler) == handlerType] -def _findCaller(): - f = sys._getframe(3) +def _findCallerPyAgnostic(*args, **kwargs): + FILE_BLACKLIST = ['logging/__init__.py', 'common/log/morelevels.py', 'configurelogging.py'] + f = logging.currentframe() while hasattr(f, "f_code"): co = f.f_code filename = os.path.normcase(co.co_filename) - if 'logging/__init__.py' in filename or 'common/log/morelevels.py' in filename: + if any([blacklisted in filename for blacklisted in FILE_BLACKLIST]): f = f.f_back continue - return (filename, f.f_lineno, co.co_name) - return ("(unknown file)", 0, "(unknown function)") + if sys.version_info[0] == 2: + return (filename, f.f_lineno, co.co_name) + return (filename, f.f_lineno, co.co_name, None) + if sys.version_info[0] == 2: + return ("(unknown file)", 0, "(unknown function)") + return ("(unknown file)", 0, "(unknown function)", None) def _useColorsForScreenOutput(): @@ -118,7 +127,7 @@ def _configureOutputToScreen(logger, loggerName): def _configureOutputToFile(logger, logName): if not os.path.isdir(config.LOGS_DIRECTORY): - os.makedirs(config.LOGS_DIRECTORY, mode=0777) + os.makedirs(config.LOGS_DIRECTORY, mode=0o777) handler = logging.FileHandler(filename=logFilename(logName)) atexit.register(handler.close) handler.setFormatter(machinereadableformatter.MachineReadableFormatter()) @@ -128,7 +137,8 @@ def _configureOutputToFile(logger, logName): logger.addHandler(handler) global _registered_file_handles _registered_file_handles[logName] = (logger, handler) - logger.findCaller = _findCaller + logger.findCaller = _findCallerPyAgnostic + def _configureLogLevels(name): if config.LOG_CONFIGURATION is not None: @@ -140,37 +150,42 @@ def _configureLogLevels(name): try: with open(config.LOGS_CONFIGURATION_OVERRIDE_FILE, 'rt') as f: overrides = json.load(f) - except: #pylint: disable=bare-except + except: # pylint: disable=bare-except overrides = {} default_overrides = overrides.get('default_log_overrides', {}) dictConfig.update(default_overrides) dictConfig.update(overrides.get(name, {})) logging.config.dictConfig(dictConfig) + def reopenLogginFiles(): global _registered_file_handles save_handles = _registered_file_handles _registered_file_handles = dict() - for logName, (logger, handler) in save_handles.iteritems(): + for logName, (logger, handler) in save_handles.items(): handler.close() logger.removeHandler(handler) _configureOutputToFile(logger, logName) + def reloadLoggingConfiguration(): reopenLogginFiles() global _name if _name is not None: _configureLogLevels(_name) + def _handleLoggingConfigurationSignal(signal, stackFrame): reloadLoggingConfiguration() + def _getMultipleFuncsHandler(funcs): def _multipleFuncsHandler(signalNumber, stackFrame): for func in funcs: func(signalNumber, stackFrame) return _multipleFuncsHandler + def _configureLoggingSignalHandlers(): currentHandler = signal.getsignal(config.UPDATE_LOGGING_CONFIGURATION_SIGNAL) if currentHandler in [signal.SIG_IGN, signal.SIG_DFL, None]: diff --git a/py/strato/common/log/lineparse.py b/py/strato/common/log/lineparse.py index 9d4c2f3..e576780 100644 --- a/py/strato/common/log/lineparse.py +++ b/py/strato/common/log/lineparse.py @@ -23,9 +23,9 @@ def seperateTimestamp(message, timestampFormat=DEFAULT_TIMESTAMP_REGEX): def translateToEpoch(timeStamp, timestampFormat=DEFAULT_TIMESTAMP_FORMAT): ms = 0 if timestampFormat.endswith('%f'): # handle milliseconds - separator = timestampFormat[-3] - if separator in timeStamp: - ms = timeStamp.rsplit(separator, 1)[1] + separator = timestampFormat[-3] + if separator in timeStamp: + ms = timeStamp.rsplit(separator, 1)[1] timeObject = datetime.datetime.strptime(timeStamp, timestampFormat) return calendar.timegm(timeObject.timetuple()) + float(ms)/1000 diff --git a/py/strato/common/log/log2text.py b/py/strato/common/log/log2text.py index 8845b40..2f74a2e 100755 --- a/py/strato/common/log/log2text.py +++ b/py/strato/common/log/log2text.py @@ -1,23 +1,25 @@ #!/usr/bin/env python -from past.builtins import long -import json -import os -import sys -import time -import signal +import strato.common.log.morelevels # relies on side effects +from strato.common.log import lineparse +from datetime import datetime +import dateparser +import argparse import logging +import signal +import socket +import select +import json import yaml +import time import glob -import strato.common.log.morelevels -import socket -import subprocess -import re import gzip -import select -import dateparser -from future.utils import iteritems -from datetime import datetime -from strato.common.log import lineparse +import sys +import os +import re +try: + import subprocess32 as subprocess +except ImportError: + import subprocess RED = '\033[31m' @@ -57,7 +59,8 @@ class Formatter(object): converter = time.gmtime def __init__(self, relativeTime, withThreads, showFullPaths, minimumLevel, microsecondPrecision, noColors, - utc=False, sinceTime=None, untilTime="01/01/2025", elapsedTime=False, shouldPrintKv=False, process=None): + utc=False, sinceTime=None, untilTime="01/01/2025", elapsedTime=False, shouldPrintKv=False, + process=None): try: self.configFile = yaml.load(open(LOG_CONFIG_FILE_PATH, 'r').read()) if self.configFile['defaultTimezone'] is not None: @@ -196,8 +199,8 @@ def _process_go_logs(self, parsed_line): caller = self._extract_go_fields(parsed_line, extra_data, 'caller') if caller is not None: file = caller.get('File', '') - line = caller.get('Line','') - func_name = caller.get('Name',None) + line = caller.get('Line', '') + func_name = caller.get('Name', None) path = '{}:{} {}'.format(file, line, func_name) else: path = parsed_line.pop('path', 'no-path') @@ -226,10 +229,10 @@ def _process_go_logs(self, parsed_line): # add key-value fields extra_data.update(parsed_line) message += '\t' - message += ', '.join(['{}={}'.format(k, v) for k, v in iteritems(extra_data) if v != '']) + message += ', '.join(['{}={}'.format(k, v) for k, v in extra_data.items() if v != '']) # add and colorize the file name message = self._add_color(message, GRAY) - message += ' ({})'.format(re.sub(r"^/", "", path)) + message += ' ({})'.format(re.sub(r"^/", "", path)) message = self._add_color(message, NORMAL_COLOR) return message, time.mktime(ts.timetuple()) @@ -275,7 +278,7 @@ def _relativeClock(self, created): return self._relativeClockFormat % (created - self._firstClock) def _absoluteClock(self, created): - msec = (created - long(created)) * 1000 + msec = (created - int(created)) * 1000 return '%s.%.03d' % (time.strftime(TIME_FORMAT, self.converter(created)), msec) @@ -312,7 +315,11 @@ def printLog(logFile, formatter, follow, tail): def _addLogName(line, colorCode, logFile, useColors): - return "%s %s(%s)%s" % (line, colorCode if useColors else '', os.path.basename(logFile), COLOR_OFF if useColors else '') + colorStart = colorCode if useColors else '' + colorEnd = COLOR_OFF if useColors else '' + filename = os.path.basename(logFile) + return "%s %s(%s)%s" % (line, colorStart, filename, colorEnd) + def _getNextParsableEntry(inputStream, logFile, colorCode, formatter): """ @@ -324,12 +331,16 @@ def _getNextParsableEntry(inputStream, logFile, colorCode, formatter): try: line = inputStream.next() formatted, timestamp = formatter.process(line, logTypeConf) - return None if formatted is None else _addLogName(formatted, colorCode, logFile, formatter.useColors), timestamp + if formatted is None: + return None + else: + return _addLogName(formatted, colorCode, logFile, formatter.useColors), timestamp except StopIteration: return None except: return line, HIGHEST_PRIORITY + def _getColorCode(id): return MULTY_LOG_COLORS[id % (len(MULTY_LOG_COLORS) - 1)] @@ -337,8 +348,8 @@ def _getColorCode(id): def printLogs(logFiles, formatter, tail): inputStreams = [(universalOpen(logFile, 'r', tail=tail), logFile) for logFile in logFiles] - currentLines= [_getNextParsableEntry(inputStream, logFile, _getColorCode(streamId), formatter) - for streamId, (inputStream, logFile) in enumerate(inputStreams)] + currentLines = [_getNextParsableEntry(inputStream, logFile, _getColorCode(streamId), formatter) + for streamId, (inputStream, logFile) in enumerate(inputStreams)] while any(currentLines): _, nextStreamId, formatted = min((line[1], streamId, line[0]) @@ -420,7 +431,7 @@ def runRemotely(host, ignoreArgs): possibleResolveSuffixes = configFile.get("possibleResolveSuffixes", []) hostname = findHostname(host, possibleResolveSuffixes) - if hostname == None: + if hostname is None: print("No reachable host was found for %s" % host) return 1 @@ -500,12 +511,13 @@ def _getDefaultPaths(self): for filePath in self.confFile['defaultPaths']: self.defaultPaths.extend(glob.glob(filePath)) + def executeRemotely(args): - if args.user != None: + if args.user is not None: updateConfFile('defaultRemoteUser', args.user) ignoreArgs.extend(['-u', '--user', args.user]) - if args.password != None: + if args.password is not None: updateConfFile('defaultRemotePassword', args.password) ignoreArgs.extend(['-p', '--password', args.password]) @@ -524,8 +536,8 @@ def copyLogFilesFromRemotes(args): try: output = subprocess.check_output(command, stderr=subprocess.STDOUT, stdin=open('/dev/null'), close_fds=True) except subprocess.CalledProcessError as exc: - print("Failed in copying files from remotes. Command : %s\n Return Code : %s\n Exception : %s" % \ - (command, exc.returncode, exc.output)) + parsedError = "Command : %s\n Return Code : %s\n Exception : %s" % (command, exc.returncode, exc.output) + print("Failed in copying files from remotes. %s" % parsedError) raise logFiles = output.splitlines() else: @@ -550,42 +562,50 @@ def minimumLevel(minLevel, noDebug): 'progress': logging.PROGRESS, 'success': logging.SUCCESS, 'step': logging.STEP} - for string, level in iteritems(level_dict): + for string, level in level_dict.items(): if string.startswith(minLevel.lower()): return level return logging.DEBUG -if __name__ == "__main__": - import argparse +def main(): parser = argparse.ArgumentParser() parser.add_argument("logFiles", metavar='logFile', nargs='*', help='logfiles to read') parser.add_argument('-d', "--noDebug", action='store_true', help='filter out debug messages') - parser.add_argument('-l', '--min-level', action='store', default='', metavar='LEVEL', help='minimal log level to display (substring is OK, case-insensitive)') + parser.add_argument('-l', '--min-level', action='store', default='', metavar='LEVEL', + help='minimal log level to display (substring is OK, case-insensitive)') parser.add_argument('-r', "--relativeTime", action='store_true', help='print relative time, not absolute') - parser.add_argument('-e', "--elapsedTime", action='store_true', help='print elaped time between each log line, not absolute') + parser.add_argument('-e', "--elapsedTime", action='store_true', + help='print elaped time between each log line, not absolute') parser.add_argument('-C', "--noColors", action='store_true', help='force monochromatic output even on a TTY') - parser.add_argument('-L', - "--noLess", action="store_true", help='Do not pipe into less even when running in a TTY') - parser.add_argument('-m', - "--microsecondPrecision", action="store_true", - help='print times in microsecond precision (instead of millisecond percision)') - parser.add_argument("-P", - "--showFullPaths", action='store_true', - help='show full path to files instead of just module and function') + parser.add_argument('-L', "--noLess", action="store_true", + help='Do not pipe into less even when running in a TTY') + parser.add_argument('-m', "--microsecondPrecision", action="store_true", + help='print times in microsecond precision (instead of millisecond percision)') + parser.add_argument("-P", "--showFullPaths", action='store_true', + help='show full path to files instead of just module and function') parser.add_argument("--withThreads", action="store_true", help='print process and thread name') - parser.add_argument("-f", "--follow", action="store_true", help='follow file forever. Force --noLess', default=False) - parser.add_argument("-U", "--utc", action="store_true", help='print logs in UTC time (default: localtime)', default=False) + parser.add_argument("-f", "--follow", action="store_true", default=False, + help='follow file forever. Force --noLess') + parser.add_argument("-U", "--utc", action="store_true", default=False, + help='print logs in UTC time (default: localtime)') parser.add_argument("--setLocaltimeOffset", type=int, metavar='HOURS', help='set custom localtime offset in hours') - parser.add_argument("-i", "--ignoreExtensions", nargs="+", metavar='EXT', help="list extensions that you don\'t want to read", default=[".gz"]) + parser.add_argument("-i", "--ignoreExtensions", nargs="+", metavar='EXT', default=[".gz"], + help="list extensions that you don\'t want to read") parser.add_argument("-a", "--showAll", action="store_true", help='show all logs', default=False) - parser.add_argument("-n", "--node", type=str, help='run strato-log on remote node (possible input is host name or service with DNS resolve', default=None) - parser.add_argument("-p", "--password", type=str, help='set default remote password to connect with', default=None) - parser.add_argument("-u", "--user", type=str, help='set default remote user to connect with', default=None) - parser.add_argument("--restoreLocaltimeOffset", action="store_true", help='restore localtime offset to machine\'s offset') + parser.add_argument("-n", "--node", type=str, default=None, + help='run strato-log on remote node (possible input is host name or service with DNS resolve') + parser.add_argument("-p", "--password", type=str, default=None, + help='set default remote password to connect with') + parser.add_argument("-u", "--user", type=str, default=None, + help='set default remote user to connect with') + parser.add_argument("--restoreLocaltimeOffset", action="store_true", + help='restore localtime offset to machine\'s offset') parser.add_argument("-t", "--tail", type=int, metavar='n', help='print n last lines only', default=0) - parser.add_argument("--since", type=str, metavar='DATE', help='Show entries not older than the specified date (e.g., 1h, 5m, two hours ago, 8/aug/1997)') - parser.add_argument("--until", type=str, metavar='DATE', help='Show entries not newer than the specified date (e.g., 0.5h, 4m, one hour ago)', default="01/01/2025") + parser.add_argument("--since", type=str, metavar='DATE', + help='Show entries not older than the specified date (e.g., 1h, 5m, two hours ago, 8/aug/1997)') + parser.add_argument("--until", type=str, metavar='DATE', default="01/01/2025", + help='Show entries not newer than the specified date (e.g., 0.5h, 4m, one hour ago)') parser.add_argument("--all-nodes", action='store_true', help='Bring asked logs from all nodes and open them') parser.add_argument("--cached", action='store_true', help='Find logs in /var/log/inspector/strato_log') parser.add_argument("-k", "--kv", action='store_true', help='print key-values in log output') @@ -594,7 +614,7 @@ def minimumLevel(minLevel, noDebug): args, unknown = parser.parse_known_args() ignoreArgs = [] - if select.select([sys.stdin,],[],[],0.0)[0]: + if select.select([sys.stdin, ], [], [], 0.0)[0]: stdin = True else: stdin = False @@ -603,7 +623,7 @@ def minimumLevel(minLevel, noDebug): print('No input was provided') exit(1) - if args.node != None: + if args.node is not None: exit(executeRemotely(args)) elif unknown: @@ -611,10 +631,10 @@ def minimumLevel(minLevel, noDebug): print("Not a valid argument \"%s\"" % arg) exit(1) - if args.setLocaltimeOffset != None: + if args.setLocaltimeOffset is not None: updateConfFile('defaultTimezone', args.setLocaltimeOffset) - if args.restoreLocaltimeOffset == True: + if args.restoreLocaltimeOffset is True: updateConfFile('defaultTimezone', None) if args.follow: @@ -622,8 +642,9 @@ def minimumLevel(minLevel, noDebug): if not args.noLess: args = " ".join(["'%s'" % a for a in sys.argv[1:]]) + lessPipe = "less -r --quit-if-one-screen --RAW-CONTROL-CHARS" result = os.system( - "python -m strato.common.log.log2text %s --noLess | less -r --quit-if-one-screen --RAW-CONTROL-CHARS" % args) + "python -m strato.common.log.log2text %s --noLess | %s" % (args, lessPipe)) sys.exit(result) # If in follow mode, print at least 10 lines @@ -660,3 +681,7 @@ def _exitOrderlyOnCtrlC(signal, frame): printLog(logFile=logFiles, formatter=formatter, follow=args.follow, tail=args.tail) else: printLogs(logFiles=logFiles, formatter=formatter, tail=args.tail) + + +if __name__ == "__main__": + main() diff --git a/py/strato/common/log/machinereadableformatter.py b/py/strato/common/log/machinereadableformatter.py index 3019061..aa0f982 100644 --- a/py/strato/common/log/machinereadableformatter.py +++ b/py/strato/common/log/machinereadableformatter.py @@ -1,4 +1,4 @@ -import simplejson +import json class MachineReadableFormatter: @@ -9,7 +9,7 @@ def format(self, record): for attribute in self._IGNORED_ATTRIBUTES: if attribute in data: del data[attribute] - return simplejson.dumps(data, default=self._defaultSerializer, encoding='raw-unicode-escape') + return json.dumps(data, default=self._defaultSerializer, ensure_ascii=False) def _defaultSerializer(self, obj): return str(obj) diff --git a/py/strato/common/log/tests/fakeuser.py b/py/strato/common/log/tests/fakeuser.py index c117967..3be901a 100644 --- a/py/strato/common/log/tests/fakeuser.py +++ b/py/strato/common/log/tests/fakeuser.py @@ -1,7 +1,10 @@ -import subprocess import tempfile import shutil import os +try: + import subprocess32 as subprocess +except ImportError: + import subprocess _tempDir = None @@ -26,16 +29,18 @@ def tempDir(): def readLogContents(logName): - return open(os.path.join(_tempDir, '%s.stratolog' % logName)).read() + with open(os.path.join(_tempDir, '%s.stratolog' % logName)) as f: + return f.read() class FakeUser: - def __init__(self, program): + def __init__(self, program_path): assert _tempDir is not None try: self._output = subprocess.check_output( - ['python', '-c', program], stderr=subprocess.STDOUT, close_fds=True, - env=dict(os.environ, STRATO_CONFIG_LOGGING="LOGS_DIRECTORY='%s'" % _tempDir)) + ['python2.7', program_path], stderr=subprocess.STDOUT, close_fds=True, + env=dict(os.environ, PYTHONPATH='./py', STRATO_CONFIG_LOGGING="LOGS_DIRECTORY='%s'" % _tempDir)) \ + .decode() except subprocess.CalledProcessError as e: print(e.output) raise diff --git a/py/strato/common/log/tests/log2textwrapper.py b/py/strato/common/log/tests/log2textwrapper.py index ea445f7..0a24745 100644 --- a/py/strato/common/log/tests/log2textwrapper.py +++ b/py/strato/common/log/tests/log2textwrapper.py @@ -1,10 +1,14 @@ import fakeuser -import subprocess +try: + import subprocess32 as subprocess +except ImportError: + import subprocess import os def run(logName): filename = os.path.join(fakeuser.tempDir(), '%s.stratolog' % logName) - output = subprocess.check_output( - ['python', '-m', 'strato.common.log.log2text', filename], stderr=subprocess.STDOUT, close_fds=True) - return output + command = ['python2.7', '-m', 'strato.common.log.log2text', filename] + output = subprocess.check_output(command, stderr=subprocess.STDOUT, + env=dict(PYTHONPATH="./py"), close_fds=True) + return output.decode() diff --git a/py/strato/common/log/tests/test.py b/py/strato/common/log/tests/test.py index f409734..cb2dd18 100644 --- a/py/strato/common/log/tests/test.py +++ b/py/strato/common/log/tests/test.py @@ -1,6 +1,9 @@ import unittest import fakeuser import log2textwrapper +from strato.common.log import configurelogging +configurelogging.configureLogging('test-strato-log', forceDirectory=".") +import logging # noqa: E402 class Test(unittest.TestCase): @@ -10,34 +13,21 @@ def setUp(self): def tearDown(self): fakeuser.tearDown() + def test_import(self): + import strato.common.log.log2text + def test_useCaseFromCommandLine(self): - PROGRAM = "\n".join([ - """from strato.common.log import configurelogging""", - """configurelogging.configureLogging('fakeuser')""", - """import logging""", - """logging.info("dict %%(here)s", dict(here='there'))""", - """logging.info('write this message')""", - ""]) - - user = fakeuser.FakeUser(PROGRAM) + PROGRAM_PATH = 'py/strato/common/log/tests/test_scripts/command_line.py' + user = fakeuser.FakeUser(PROGRAM_PATH) self.assertTrue('write this message' in user.output()) self.assertTrue('write this message' in fakeuser.readLogContents('fakeuser')) self.assertTrue('write this message' in log2textwrapper.run('fakeuser')) def test_useCaseWithSublogger(self): - PROGRAM = "\n".join([ - """from strato.common.log import configurelogging""", - """configurelogging.configureLogging('mainlog')""", - """configurelogging.configureLogger('sub.logger')""", - """import logging""", - """logging.info('write this message')""", - """logging.getLogger('sub.logger').info('sub message')""", - ""]) - - user = fakeuser.FakeUser(PROGRAM) + PROGRAM_PATH = 'py/strato/common/log/tests/test_scripts/sublogger.py' + user = fakeuser.FakeUser(PROGRAM_PATH) self.assertTrue('write this message' in user.output()) self.assertTrue('sub message' in user.output()) - self.assertTrue('write this message' in fakeuser.readLogContents('mainlog')) self.assertTrue('write this message' in log2textwrapper.run('mainlog')) self.assertTrue('sub message' not in fakeuser.readLogContents('mainlog')) @@ -49,20 +39,8 @@ def test_useCaseWithSublogger(self): self.assertTrue('write this message' not in log2textwrapper.run('mainlog__sub.logger')) def test_useCaseForDaemon(self): - PROGRAM = "\n".join([ - """import strato.common.log.environment""", - """strato.common.log.environment.guessIfRunningAsAService = lambda: True""", - """from strato.common.log import configurelogging""", - """configurelogging.configureLogging('mainlog')""", - """configurelogging.configureLogger('sub.logger')""", - """import logging""", - """logging.info('write this message')""", - """logging.error('root error message')""", - """logging.getLogger('sub.logger').info('sub message')""", - """logging.getLogger('sub.logger').error('sub error message')""", - ""]) - - user = fakeuser.FakeUser(PROGRAM) + PROGRAM_PATH = 'py/strato/common/log/tests/test_scripts/daemon.py' + user = fakeuser.FakeUser(PROGRAM_PATH) self.assertTrue('write this message' not in user.output()) self.assertTrue('sub message' in user.output()) self.assertTrue('root error message' in user.output()) @@ -86,37 +64,9 @@ def test_useCaseForDaemon(self): self.assertTrue('root error message' not in fakeuser.readLogContents('mainlog__sub.logger')) self.assertTrue('root error message' not in log2textwrapper.run('mainlog__sub.logger')) - def test_useCaseForConfigurationChange(self): - PROGRAM = "\n".join([ - """import strato.common.log.config""", - """strato.common.log.config.LOGS_CONFIGURATION_OVERRIDE_FILE = '/tmp/test_logging'""", - """import json""", - """firstConfig = {'default_log_overrides': {'handlers': {'console': {'level': 'WARNING'}, 'file': {'level': 'INFO'}}}}""", - """with open(strato.common.log.config.LOGS_CONFIGURATION_OVERRIDE_FILE, 'wt') as fd:""", - """ json.dump(firstConfig, fd)""", - """from strato.common.log import configurelogging""", - """configurelogging.configureLogging('mainlog')""", - """configurelogging.configureLogger('sub.logger')""", - """import logging""", - """logging.info('write this message')""", - """logging.error('root error message')""", - """logging.getLogger('sub.logger').info('sub message')""", - """logging.getLogger('sub.logger').error('sub error message')""", - """secondConfig = {'mainlog': {'loggers': {'sub.logger': {'level': 'WARNING', 'propagate' : False}},"""\ - """'handlers': {'console': {'level': 'INFO'}, 'file': {'level': 'INFO'}}, 'root': {'level': 'INFO'}, 'incremental' : True}}""", - """with open(strato.common.log.config.LOGS_CONFIGURATION_OVERRIDE_FILE, 'wt') as fd:""", - """ json.dump(secondConfig, fd)""", - """import os""", - """import signal""", - """os.kill(os.getpid(), strato.common.log.config.UPDATE_LOGGING_CONFIGURATION_SIGNAL)""", - """logging.getLogger().info('message after config change')""", - """logging.error('root error after config change')""", - """logging.getLogger('sub.logger').info('sub after config change')""", - """logging.getLogger('sub.logger').error('sub error after config change')""", - ""]) - - user = fakeuser.FakeUser(PROGRAM) + PROGRAM_PATH = 'py/strato/common/log/tests/test_scripts/config_change.py' + user = fakeuser.FakeUser(PROGRAM_PATH) self.assertTrue('write this message' not in user.output()) self.assertTrue('write this message' in fakeuser.readLogContents('mainlog')) self.assertTrue('root error message' in user.output()) @@ -135,5 +85,23 @@ def test_useCaseForConfigurationChange(self): self.assertTrue('sub error after config change' in user.output()) self.assertTrue('sub error after config change' in fakeuser.readLogContents('mainlog__sub.logger')) -if __name__ == '__main__': + +def main(): + logging.warning('Running test') + logging.error('Running test') + logging.progress('Running test') + logging.step('Running test') + logging.critical('Running test') + logging.success('Running test') + logging.debug('Running test') + try: + raise Exception('Test exception') + except Exception: + logging.exception('Running test') unittest.main() + + +logging.info('Imported test') + +if __name__ == '__main__': + main() diff --git a/py/strato/common/log/tests/test_scripts/command_line.py b/py/strato/common/log/tests/test_scripts/command_line.py new file mode 100644 index 0000000..4c19b6c --- /dev/null +++ b/py/strato/common/log/tests/test_scripts/command_line.py @@ -0,0 +1,5 @@ +from strato.common.log import configurelogging +configurelogging.configureLogging('fakeuser') +import logging # noqa: E402 +logging.info("dict %%(here)s", dict(here='there')) +logging.info('write this message') diff --git a/py/strato/common/log/tests/test_scripts/config_change.py b/py/strato/common/log/tests/test_scripts/config_change.py new file mode 100644 index 0000000..34cc56a --- /dev/null +++ b/py/strato/common/log/tests/test_scripts/config_change.py @@ -0,0 +1,27 @@ +import strato.common.log.config +strato.common.log.config.LOGS_CONFIGURATION_OVERRIDE_FILE = '/tmp/test_logging' +import json # noqa: E402 +firstConfig = {'default_log_overrides': + {'handlers': {'console': {'level': 'WARNING'}, 'file': {'level': 'INFO'}}}} +with open(strato.common.log.config.LOGS_CONFIGURATION_OVERRIDE_FILE, 'wt') as fd: + json.dump(firstConfig, fd) +from strato.common.log import configurelogging # noqa: E402 +configurelogging.configureLogging('mainlog') +configurelogging.configureLogger('sub.logger') +import logging # noqa: E402 +logging.info('write this message') +logging.error('root error message') +logging.getLogger('sub.logger').info('sub message') +logging.getLogger('sub.logger').error('sub error message') +secondConfig = {'mainlog': {'loggers': {'sub.logger': {'level': 'WARNING', 'propagate': False}}, + 'handlers': {'console': {'level': 'INFO'}, 'file': {'level': 'INFO'}}, + 'root': {'level': 'INFO'}, 'incremental': True}} +with open(strato.common.log.config.LOGS_CONFIGURATION_OVERRIDE_FILE, 'wt') as fd: + json.dump(secondConfig, fd) + +import os # noqa: E402 +os.kill(os.getpid(), strato.common.log.config.UPDATE_LOGGING_CONFIGURATION_SIGNAL) +logging.getLogger().info('message after config change') +logging.error('root error after config change') +logging.getLogger('sub.logger').info('sub after config change') +logging.getLogger('sub.logger').error('sub error after config change') diff --git a/py/strato/common/log/tests/test_scripts/daemon.py b/py/strato/common/log/tests/test_scripts/daemon.py new file mode 100644 index 0000000..6305e25 --- /dev/null +++ b/py/strato/common/log/tests/test_scripts/daemon.py @@ -0,0 +1,10 @@ +import strato.common.log.environment +strato.common.log.environment.guessIfRunningAsAService = lambda: True +from strato.common.log import configurelogging # noqa: E402 +configurelogging.configureLogging('mainlog') +configurelogging.configureLogger('sub.logger') +import logging # noqa: E402 +logging.info('write this message') +logging.error('root error message') +logging.getLogger('sub.logger').info('sub message') +logging.getLogger('sub.logger').error('sub error message') diff --git a/py/strato/common/log/tests/test_scripts/sublogger.py b/py/strato/common/log/tests/test_scripts/sublogger.py new file mode 100644 index 0000000..c793ee2 --- /dev/null +++ b/py/strato/common/log/tests/test_scripts/sublogger.py @@ -0,0 +1,7 @@ +from strato.common.log import configurelogging +configurelogging.configureLogging('mainlog') +configurelogging.configureLogger('sub.logger') + +import logging # noqa: E402 +logging.info('write this message') +logging.getLogger('sub.logger').info('sub message') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..38b9357 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "python-setuptools-plugins", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a64878d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +python-setuptools-plugins==0.0.2+g865898c +pyyaml +regex<2022; python_version < "3" +dateparser diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e0d8b5f --- /dev/null +++ b/setup.py @@ -0,0 +1,32 @@ +from setuptools_plugins.version import get_version +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="strato_common_log", + packages=setuptools.find_packages(where="py"), + package_dir={"": "py"}, + version=get_version(), + entry_points={ + 'console_scripts': [ + 'strato-log = strato.common.log.log2text:main', + ], + }, + author="Stratoscale", + author_email="zcompute@zadarastorage.com", + description="", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/Stratoscale/pycommonlog", + project_urls={ + "Bug Tracker": "https://github.com/Stratoscale/pycommonlog/issues", + }, + classifiers=[ + "Programming Language :: Python :: 2", + "License :: Apache License 2.0", + "Operating System :: OS Independent", + ], + options={'bdist_wheel': {'universal': True}} +) diff --git a/skipper.yaml b/skipper.yaml new file mode 100644 index 0000000..b669146 --- /dev/null +++ b/skipper.yaml @@ -0,0 +1,5 @@ +registry: rackattack-nas.dc1:5000 +build-container-image: pycommonlog-build + +make: + makefile: Makefile