Skip to content

Commit

Permalink
RD-1601 py3 py2 agnostic
Browse files Browse the repository at this point in the history
fix encoding issues
semver support
skipperize
remove upseto
  • Loading branch information
IliaFeldgun committed Jun 20, 2024
1 parent d08bdcc commit 3f4cd59
Show file tree
Hide file tree
Showing 24 changed files with 327 additions and 156 deletions.
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
*.pyc
*.swp
.idea
__pycache__
dist
build
*.egg-info
*.stratolog
8 changes: 8 additions & 0 deletions Dockerfile.pycommonlog-build
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM rackattack-nas.dc1:5000/ubuntu-dev-base:379308aa77bd0a19ebf12d868e7d19ee35a19d6c
RUN apt-get install -y python2.7 python2.7-dev
RUN curl 'https://bootstrap.pypa.io/pip/2.7/get-pip.py' | python2.7
COPY requirements.txt /tmp/requirements.txt
COPY dev-requirements.txt /tmp/dev-requirements.txt
RUN python2.7 -m pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt
RUN rm /tmp/requirements.txt /tmp/dev-requirements.txt
32 changes: 27 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,29 @@
all: test check_convention
.PHONY: all test build test_py2 test_py3 lint_py2 lint_py3
ARTIFACT=dist/strato_common_log-*-py2.py3-none-any.whl
all: lint test build

test:
PYTHONPATH=$(PWD)/py python py/strato/common/log/tests/test.py
test: test_py2 test_py3

check_convention:
pep8 py --max-line-length=109
test_py3:
PYTHONPATH=$(PWD)/py python3 py/strato/common/log/tests/test.py

test_py2:
PYTHONPATH=$(PWD)/py python2.7 py/strato/common/log/tests/test.py

lint: lint_py2 lint_py3

lint_py3:
python3 -m pep8 py --max-line-length=120

lint_py2:
python2.7 -m pep8 py --max-line-length=120

build: $(ARTIFACT)

clean:
find . -name *.pyc -delete
find . -name __pycache__ -delete
rm -rf dist */*.egg-info build *.stratolog

$(ARTIFACT):
python3 -m build --wheel
25 changes: 25 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# pycommonlog

# Build
```bash
skipper make -i all
```

# Demo

```python
from strato.common.log import configurelogging
configurelogging.configureLogging('test-strato-log', forceDirectory=".")
import logging # noqa: E402
logging.warning('Running test')
logging.error('Running test')
logging.progress('Running test')
logging.step('Running test')
logging.critical('Running test')
logging.success('Running test')
logging.debug('Running test')
try:
raise Exception('Test exception')
except Exception:
logging.exception('Running test')
```
2 changes: 2 additions & 0 deletions artifacts.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pips:
- dist/strato_common_log-*-py2.py3-none-any.whl
3 changes: 3 additions & 0 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
build
pep8
ipdb
2 changes: 0 additions & 2 deletions py/strato/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +0,0 @@
import upseto.pythonnamespacejoin
__path__.extend(upseto.pythonnamespacejoin.join(globals()))
2 changes: 0 additions & 2 deletions py/strato/common/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +0,0 @@
import upseto.pythonnamespacejoin
__path__.extend(upseto.pythonnamespacejoin.join(globals()))
2 changes: 1 addition & 1 deletion py/strato/common/log/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,4 @@
"incremental": True
}

exec os.environ.get("STRATO_CONFIG_LOGGING", "")
exec(os.environ.get("STRATO_CONFIG_LOGGING", ""))
35 changes: 25 additions & 10 deletions py/strato/common/log/configurelogging.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,16 @@
import sys
import atexit
import json
import subprocess
import signal
try:
import subprocess32 as subprocess
except ImportError:
import subprocess

_name = None
_registered_file_handles = dict()


def logFilename(name):
return '%s/%s%s' % (config.LOGS_DIRECTORY, name, config.LOGS_SUFFIX)

Expand Down Expand Up @@ -77,16 +81,21 @@ def changeHandlerLogLevelbyHandlerType(logger, logLevel, handlerType=None):
[handler.setLevel(logLevel) for handler in logger.handlers if not handlerType or type(handler) == handlerType]


def _findCaller():
f = sys._getframe(3)
def _findCallerPyAgnostic(*args, **kwargs):
FILE_BLACKLIST = ['logging/__init__.py', 'common/log/morelevels.py', 'configurelogging.py']
f = logging.currentframe()
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
if 'logging/__init__.py' in filename or 'common/log/morelevels.py' in filename:
if any([blacklisted in filename for blacklisted in FILE_BLACKLIST]):
f = f.f_back
continue
return (filename, f.f_lineno, co.co_name)
return ("(unknown file)", 0, "(unknown function)")
if sys.version_info[0] == 2:
return (filename, f.f_lineno, co.co_name)
return (filename, f.f_lineno, co.co_name, None)
if sys.version_info[0] == 2:
return ("(unknown file)", 0, "(unknown function)")
return ("(unknown file)", 0, "(unknown function)", None)


def _useColorsForScreenOutput():
Expand Down Expand Up @@ -118,7 +127,7 @@ def _configureOutputToScreen(logger, loggerName):

def _configureOutputToFile(logger, logName):
if not os.path.isdir(config.LOGS_DIRECTORY):
os.makedirs(config.LOGS_DIRECTORY, mode=0777)
os.makedirs(config.LOGS_DIRECTORY, mode=0o777)
handler = logging.FileHandler(filename=logFilename(logName))
atexit.register(handler.close)
handler.setFormatter(machinereadableformatter.MachineReadableFormatter())
Expand All @@ -128,7 +137,8 @@ def _configureOutputToFile(logger, logName):
logger.addHandler(handler)
global _registered_file_handles
_registered_file_handles[logName] = (logger, handler)
logger.findCaller = _findCaller
logger.findCaller = _findCallerPyAgnostic


def _configureLogLevels(name):
if config.LOG_CONFIGURATION is not None:
Expand All @@ -140,37 +150,42 @@ def _configureLogLevels(name):
try:
with open(config.LOGS_CONFIGURATION_OVERRIDE_FILE, 'rt') as f:
overrides = json.load(f)
except: #pylint: disable=bare-except
except: # pylint: disable=bare-except
overrides = {}
default_overrides = overrides.get('default_log_overrides', {})
dictConfig.update(default_overrides)
dictConfig.update(overrides.get(name, {}))
logging.config.dictConfig(dictConfig)


def reopenLogginFiles():
global _registered_file_handles
save_handles = _registered_file_handles
_registered_file_handles = dict()
for logName, (logger, handler) in save_handles.iteritems():
for logName, (logger, handler) in save_handles.items():
handler.close()
logger.removeHandler(handler)
_configureOutputToFile(logger, logName)


def reloadLoggingConfiguration():
reopenLogginFiles()
global _name
if _name is not None:
_configureLogLevels(_name)


def _handleLoggingConfigurationSignal(signal, stackFrame):
reloadLoggingConfiguration()


def _getMultipleFuncsHandler(funcs):
def _multipleFuncsHandler(signalNumber, stackFrame):
for func in funcs:
func(signalNumber, stackFrame)
return _multipleFuncsHandler


def _configureLoggingSignalHandlers():
currentHandler = signal.getsignal(config.UPDATE_LOGGING_CONFIGURATION_SIGNAL)
if currentHandler in [signal.SIG_IGN, signal.SIG_DFL, None]:
Expand Down
6 changes: 3 additions & 3 deletions py/strato/common/log/lineparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ def seperateTimestamp(message, timestampFormat=DEFAULT_TIMESTAMP_REGEX):
def translateToEpoch(timeStamp, timestampFormat=DEFAULT_TIMESTAMP_FORMAT):
ms = 0
if timestampFormat.endswith('%f'): # handle milliseconds
separator = timestampFormat[-3]
if separator in timeStamp:
ms = timeStamp.rsplit(separator, 1)[1]
separator = timestampFormat[-3]
if separator in timeStamp:
ms = timeStamp.rsplit(separator, 1)[1]
timeObject = datetime.datetime.strptime(timeStamp, timestampFormat)
return calendar.timegm(timeObject.timetuple()) + float(ms)/1000

Expand Down
Loading

0 comments on commit 3f4cd59

Please sign in to comment.