From 06035a80a66a5eba843511852261cdae842b3172 Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 5 Feb 2019 20:46:19 -0600 Subject: [PATCH 01/20] switch from travis-ci to jenkins --- .travis.yml | 18 --------- Jenkinsfile | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ deploy.sh | 79 ------------------------------------- prebuild.sh | 3 -- 4 files changed, 111 insertions(+), 100 deletions(-) delete mode 100644 .travis.yml create mode 100644 Jenkinsfile delete mode 100644 deploy.sh delete mode 100644 prebuild.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0dd05185..00000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -sudo: true -dist: xenial -language: minimal -services: - - docker -addons: - apt: - packages: - - docker-ce -deploy: - - provider: script - script: bash deploy.sh - on: - branch: master - - provider: script - script: bash deploy.sh - on: - branch: develop \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..85ad7f04 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,111 @@ +pipeline { + agent none + environment { + REPOSITORY = "boerderij/varken" + } + stages { + stage('Flake8') { + agent { label 'amd64'} + steps { + sh ''' + python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 *.py varken/*.py + rm -rf venv/ + ''' + } + } + stage('Docker Builds') { + parallel { + stage('amd64') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'amd64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + def image = docker.build("${REPOSITORY}:${tag}-amd64") + image.push() + + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${REPOSITORY}:develop-amd64") + image.push() + } + } + } + } + stage('ARMv6') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'arm64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + def image = docker.build("${REPOSITORY}:${tag}-arm", "-f Dockerfile.arm .") + image.push() + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${REPOSITORY}:develop-arm", "-f Dockerfile.arm .") + image.push() + } + } + } + } + stage('ARM64v8') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'arm64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + def image = docker.build("${REPOSITORY}:${tag}-arm64", "-f Dockerfile.arm64 .") + image.push() + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${REPOSITORY}:develop-arm64", "-f Dockerfile.arm64 .") + image.push() + } + } + } + } + } + } + stage('Docker Manifest Build') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'amd64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + sh(script: "docker manifest create ${REPOSITORY}:${tag} ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") + sh(script: "docker manifest inspect ${REPOSITORY}:${tag}") + sh(script: "docker manifest push -p ${REPOSITORY}:${tag}") + sh(script: "docker manifest create ${REPOSITORY}:latest ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") + sh(script: "docker manifest inspect ${REPOSITORY}:latest") + sh(script: "docker manifest push -p ${REPOSITORY}:latest") + } else if (BRANCH_NAME == 'develop') { + sh(script: "docker manifest create ${REPOSITORY}:develop ${REPOSITORY}:develop-amd64 ${REPOSITORY}:develop-arm64 ${REPOSITORY}:develop-arm") + sh(script: "docker manifest inspect ${REPOSITORY}:develop") + sh(script: "docker manifest push -p ${REPOSITORY}:develop") + } + } + } + } + } +} diff --git a/deploy.sh b/deploy.sh deleted file mode 100644 index d2feb82d..00000000 --- a/deploy.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Travis-ci convenience environment vars used: -# TRAVIS_BRANCH | branch name -# $TRAVIS_REPO_SLUG | organization/project (GitHub Capitalization) -# Travis-ci manual environment vars used: -# GITHUB_USER | github username -# GITHUB_TOKEN | $GITHUB_USER's token -# DOCKER_USER | docker username -# DOCKER_PASSWORD | $DOCKER_USER's password - -VERSION="$(grep -i version varken/__init__.py | cut -d' ' -f3 | tr -d \")" - -# Set branch to latest if master, else keep the same -if [[ "$TRAVIS_BRANCH" == "master" ]]; then - BRANCH="latest" -else - BRANCH="$TRAVIS_BRANCH" -fi - -# get the docker lowercase variant of the repo_name -REPOSITORY="$(echo $TRAVIS_REPO_SLUG | tr '[:upper:]' '[:lower:]')" - -# Docker experimental config -echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json -[[ -d ~/.docker ]] || mkdir ~/.docker -[[ -f ~/.docker/config.json ]] || touch ~/.docker/config.json -echo '{"experimental":"enabled"}' | sudo tee ~/.docker/config.json -sudo service docker restart - -# Auth -echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin - -# Prepare QEMU for ARM builds -docker run --rm --privileged multiarch/qemu-user-static:register --reset -bash prebuild.sh -chmod +x qemu-aarch64-static qemu-arm-static - -# Set tag based off of branch -if [[ "$BRANCH" == "latest" ]]; then - TAG="$VERSION" -else - TAG="$BRANCH" -fi - -# AMDx64 -docker build -t "${REPOSITORY}:${TAG}-amd64" . && \ -docker push "${REPOSITORY}:${TAG}-amd64" - -# Create Initial Manifests -docker manifest create "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-amd64" -if [[ "$BRANCH" == "latest" ]]; then - docker manifest create "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-amd64" -fi - -# ARM variants -for i in $(ls *arm*); do - ARCH="$(echo ${i} | cut -d. -f2)" - docker build -f "Dockerfile.${ARCH}" -t "${REPOSITORY}:${TAG}-${ARCH}" . && \ - docker push "${REPOSITORY}:${TAG}-${ARCH}" - # Add variant to manifest - docker manifest create -a "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" - if [[ "$BRANCH" == "latest" ]]; then - docker manifest create -a "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" - fi -done - -docker manifest inspect "${REPOSITORY}:${TAG}" && \ -docker manifest push "${REPOSITORY}:${TAG}" -if [[ "$BRANCH" == "latest" ]]; then - docker manifest inspect "${REPOSITORY}:${BRANCH}" && \ - docker manifest push "${REPOSITORY}:${BRANCH}" -fi - -# Git tags -if [[ "$BRANCH" == "latest" ]]; then - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${REPOSITORY}.git" && \ - git tag "${VERSION}" && \ - git push --tags -fi \ No newline at end of file diff --git a/prebuild.sh b/prebuild.sh deleted file mode 100644 index 8d12fd72..00000000 --- a/prebuild.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-aarch64-static" -wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-arm-static" \ No newline at end of file From 11fe44a9755e8d04b1eb6494012363549daac17a Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 5 Feb 2019 20:52:41 -0600 Subject: [PATCH 02/20] flake8 --- Varken.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Varken.py b/Varken.py index cd83c344..a47feffa 100644 --- a/Varken.py +++ b/Varken.py @@ -10,7 +10,8 @@ from argparse import ArgumentParser, RawTextHelpFormatter from logging import getLogger, StreamHandler, Formatter, DEBUG -from varken import structures # Needed to check version of python +# Needed to check version of python +from varken import structures # noqa from varken.ombi import OmbiAPI from varken.unifi import UniFiAPI from varken import VERSION, BRANCH From a77b134122b6e9bfa1545aebb97a010c5541dc41 Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 5 Feb 2019 20:54:37 -0600 Subject: [PATCH 03/20] remove dependancies --- Dockerfile.arm | 2 -- Dockerfile.arm64 | 2 -- 2 files changed, 4 deletions(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index 3bcad327..8f126177 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -6,8 +6,6 @@ ENV DEBUG="False" WORKDIR /app -COPY /qemu-arm-static /usr/bin/qemu-arm-static - COPY /requirements.txt /Varken.py /app/ COPY /varken /app/varken diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index d18a1542..98e5ff9c 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -6,8 +6,6 @@ ENV DEBUG="False" WORKDIR /app -COPY /qemu-aarch64-static /usr/bin/qemu-aarch64-static - COPY /requirements.txt /Varken.py /app/ COPY /varken /app/varken From a9a0e1f21a2e17defb160278b3433bc0ac29a126 Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 5 Feb 2019 23:47:56 -0600 Subject: [PATCH 04/20] better jenkinsfile --- Jenkinsfile | 61 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 85ad7f04..d751aef5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,16 +1,24 @@ pipeline { agent none environment { - REPOSITORY = "boerderij/varken" + DOCKER_REPO = "boerderij/varken" + GIT_REPO = 'Boerderij/Varken' + VERSION_FILE = "varken/__init__.py" + FLAKE_FILES = "Varken.py varken/*.py" + TAG = "" + GIT_TOKEN = credentials('github-jenkins-token') } stages { stage('Flake8') { agent { label 'amd64'} steps { - sh ''' - python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 *.py varken/*.py + sh """ + python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 ${FLAKE_FILES} rm -rf venv/ - ''' + """ + script { + TAG = sh(returnStdout: true, script: 'grep -i version ${VERSION_FILE} | cut -d" " -f3 | tr -d \\"').trim() + } } } stage('Docker Builds') { @@ -26,12 +34,11 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - def image = docker.build("${REPOSITORY}:${tag}-amd64") + def image = docker.build("${DOCKER_REPO}:${TAG}-amd64") image.push() } else if (BRANCH_NAME == 'develop') { - def image = docker.build("${REPOSITORY}:develop-amd64") + def image = docker.build("${DOCKER_REPO}:develop-amd64") image.push() } } @@ -48,11 +55,10 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - def image = docker.build("${REPOSITORY}:${tag}-arm", "-f Dockerfile.arm .") + def image = docker.build("${DOCKER_REPO}:${TAG}-arm", "-f Dockerfile.arm .") image.push() } else if (BRANCH_NAME == 'develop') { - def image = docker.build("${REPOSITORY}:develop-arm", "-f Dockerfile.arm .") + def image = docker.build("${DOCKER_REPO}:develop-arm", "-f Dockerfile.arm .") image.push() } } @@ -69,11 +75,10 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - def image = docker.build("${REPOSITORY}:${tag}-arm64", "-f Dockerfile.arm64 .") + def image = docker.build("${DOCKER_REPO}:${TAG}-arm64", "-f Dockerfile.arm64 .") image.push() } else if (BRANCH_NAME == 'develop') { - def image = docker.build("${REPOSITORY}:develop-arm64", "-f Dockerfile.arm64 .") + def image = docker.build("${DOCKER_REPO}:develop-arm64", "-f Dockerfile.arm64 .") image.push() } } @@ -92,20 +97,30 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - sh(script: "docker manifest create ${REPOSITORY}:${tag} ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") - sh(script: "docker manifest inspect ${REPOSITORY}:${tag}") - sh(script: "docker manifest push -p ${REPOSITORY}:${tag}") - sh(script: "docker manifest create ${REPOSITORY}:latest ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") - sh(script: "docker manifest inspect ${REPOSITORY}:latest") - sh(script: "docker manifest push -p ${REPOSITORY}:latest") + sh(script: "docker manifest create ${DOCKER_REPO}:${TAG} ${DOCKER_REPO}:${TAG}-amd64 ${DOCKER_REPO}:${TAG}-arm64 ${DOCKER_REPO}:${TAG}-arm") + sh(script: "docker manifest inspect ${DOCKER_REPO}:${TAG}") + sh(script: "docker manifest push -p ${DOCKER_REPO}:${TAG}") + sh(script: "docker manifest create ${DOCKER_REPO}:latest ${DOCKER_REPO}:${TAG}-amd64 ${DOCKER_REPO}:${TAG}-arm64 ${DOCKER_REPO}:${TAG}-arm") + sh(script: "docker manifest inspect ${DOCKER_REPO}:latest") + sh(script: "docker manifest push -p ${DOCKER_REPO}:latest") } else if (BRANCH_NAME == 'develop') { - sh(script: "docker manifest create ${REPOSITORY}:develop ${REPOSITORY}:develop-amd64 ${REPOSITORY}:develop-arm64 ${REPOSITORY}:develop-arm") - sh(script: "docker manifest inspect ${REPOSITORY}:develop") - sh(script: "docker manifest push -p ${REPOSITORY}:develop") + sh(script: "docker manifest create ${DOCKER_REPO}:develop ${DOCKER_REPO}:develop-amd64 ${DOCKER_REPO}:develop-arm64 ${DOCKER_REPO}:develop-arm") + sh(script: "docker manifest inspect ${DOCKER_REPO}:develop") + sh(script: "docker manifest push -p ${DOCKER_REPO}:develop") } } } } + stage('GitHub Release') { + when { branch 'master' } + agent { label 'amd64'} + steps { + sh """ + git remote set-url origin "https://${GIT_TOKEN_USR}:${GIT_TOKEN_PSW}@github.com/${GIT_REPO}.git" + git tag ${TAG} + git push --tags + """ + } + } } } From 7987a2c40e46c7a086f95b0f75c67f8b3a038e70 Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 5 Feb 2019 23:51:02 -0600 Subject: [PATCH 05/20] update badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 105a1804..bb61fe55 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

-[![Build Status](https://travis-ci.org/Boerderij/Varken.svg?branch=master)](https://travis-ci.org/Boerderij/Varken) +[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Varken/master)](https://jenkins.cajun.pro/job/Varken/develop) [![Discord](https://img.shields.io/discord/518970285773422592.svg?colorB=7289DA&label=Discord&logo=Discord&logoColor=7289DA&style=flat-square)](https://discord.gg/VjZ6qSM) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) From eea506f71dd3ddc3d927a673fcc5cba046f6f32e Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 12 Feb 2019 12:44:32 -0600 Subject: [PATCH 06/20] open/close file for windows issues --- varken/helpers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/varken/helpers.py b/varken/helpers.py index 93817bed..8657d43a 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -24,7 +24,14 @@ def __init__(self, data_folder): self.update() self.logger.info('Opening persistent connection to GeoLite2 DB...') - self.reader = Reader(self.dbfile) + self.reader = None + self.reader_manager(action='open') + + def reader_manager(self, action=None): + if action == 'open': + self.reader = Reader(self.dbfile) + else: + self.reader.close() def lookup(self, ipaddress): ip = ipaddress @@ -47,8 +54,10 @@ def update(self): if dbdate < first_wednesday_date < today: self.logger.info("Newer GeoLite2 DB available, Updating...") + self.reader_manager(action='close') remove(self.dbfile) self.download() + self.reader_manager(action='open') else: td = first_wednesday_date - today if td.days < 0: From d6908e1743f937276192f2c62c10a9ad936043ef Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 12 Feb 2019 12:56:40 -0600 Subject: [PATCH 07/20] change order --- varken/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 8657d43a..77fb5421 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -21,11 +21,11 @@ def __init__(self, data_folder): self.data_folder = data_folder self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb')) self.logger = getLogger() + self.reader = None + self.reader_manager(action='open') self.update() self.logger.info('Opening persistent connection to GeoLite2 DB...') - self.reader = None - self.reader_manager(action='open') def reader_manager(self, action=None): if action == 'open': From 8fab8f1ffb5b18046a1b1fa23e941a83de4146af Mon Sep 17 00:00:00 2001 From: dirtycajunrice Date: Tue, 12 Feb 2019 13:02:00 -0600 Subject: [PATCH 08/20] keep retrying geolite database download. Fixes #103 --- varken/helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 77fb5421..34ab9401 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -2,6 +2,7 @@ from datetime import date from logging import getLogger from ipaddress import IPv4Address +from urllib.error import HTTPError from calendar import monthcalendar from geoip2.database import Reader from tarfile import open as taropen @@ -68,9 +69,15 @@ def update(self): def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' + downloaded = False - self.logger.info('Downloading GeoLite2 from %s', url) - urlretrieve(url, tar_dbfile) + while not downloaded: + self.logger.info('Downloading GeoLite2 from %s', url) + try: + urlretrieve(url, tar_dbfile) + downloaded = True + except HTTPError as e: + self.logger.error('Problem downloading new MaxMind DB... Trying again. Error: %s', e) self.logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) From 603c5ee3cd60d8f1fb4ac838ef6b2bb0db8236d7 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 6 Mar 2019 08:53:36 -0800 Subject: [PATCH 09/20] Add new Tautulli structures #111 --- varken/structures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/varken/structures.py b/varken/structures.py index b2101805..cf3e984a 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -455,3 +455,5 @@ class TautulliStream(NamedTuple): width: str = None writers: list = None year: str = None + secure: str = None + relayed: str = None From f5c41473d8973846f06a89215d1bfbff90b7920d Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 12:47:13 -0400 Subject: [PATCH 10/20] Remove print in Varken.py --- Varken.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Varken.py b/Varken.py index a47feffa..1c0c7f27 100644 --- a/Varken.py +++ b/Varken.py @@ -32,7 +32,6 @@ def thread(): while schedule.jobs: job = QUEUE.get() a = job() - print(a) if a is not None: schedule.clear(a) QUEUE.task_done() From dd65fa235b9fb73ac0c7242da12cc4324a684400 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 14:08:10 -0400 Subject: [PATCH 11/20] Rework DB downloader fixes #113 --- varken/helpers.py | 54 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 34ab9401..ac39da4f 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -1,5 +1,6 @@ from hashlib import md5 -from datetime import date +from datetime import date, timedelta +from time import sleep from logging import getLogger from ipaddress import IPv4Address from urllib.error import HTTPError @@ -24,13 +25,21 @@ def __init__(self, data_folder): self.logger = getLogger() self.reader = None self.reader_manager(action='open') - self.update() self.logger.info('Opening persistent connection to GeoLite2 DB...') def reader_manager(self, action=None): if action == 'open': - self.reader = Reader(self.dbfile) + try: + self.reader = Reader(self.dbfile) + except FileNotFoundError: + self.logger.error("Could not find GeoLite2 DB! Downloading!") + result_status = self.download() + if result_status: + self.logger.error("Could not download GeoLite2 DB!!!, You may need to manually install it.") + exit(1) + else: + self.reader = Reader(self.dbfile) else: self.reader.close() @@ -45,41 +54,62 @@ def update(self): try: dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) + db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60) + except FileNotFoundError: self.logger.error("Could not find GeoLite2 DB as: %s", self.dbfile) self.download() dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) + db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60) - first_wednesday_day = [week[2:3][0] for week in monthcalendar(today.year, today.month) if week[2:3][0] != 0][0] - first_wednesday_date = date(today.year, today.month, first_wednesday_day) - if dbdate < first_wednesday_date < today: + if db_next_update < today: self.logger.info("Newer GeoLite2 DB available, Updating...") + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) self.reader_manager(action='close') - remove(self.dbfile) self.download() self.reader_manager(action='open') else: - td = first_wednesday_date - today + td = dbdate - today if td.days < 0: - self.logger.debug('Geolite2 DB is only %s days old. Keeping current copy', abs(td.days)) + self.logger.debug("Geolite2 DB is only %s days old. Keeping current copy. Next update after %s", + abs(td.days), db_next_update) + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) else: - self.logger.debug('Geolite2 DB will update in %s days', abs(td.days)) + self.logger.debug("Geolite2 DB will update in %s days", abs(td.days)) + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) + def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' downloaded = False + retry_counter = 0 + while not downloaded: self.logger.info('Downloading GeoLite2 from %s', url) try: urlretrieve(url, tar_dbfile) downloaded = True except HTTPError as e: - self.logger.error('Problem downloading new MaxMind DB... Trying again. Error: %s', e) + self.logger.error("Problem downloading new GeoLite2 DB... Trying again. Error: %s", e) + sleep(2) + retry_counter = (retry_counter + 1) + + if retry_counter >= 3: + self.logger.error("Retried downloading the new GeoLite2 DB 3 times and failed... Aborting!") + result_status = 1 + return result_status + try: + remove(self.dbfile) + except FileNotFoundError: + self.logger.warn("Cannot remove GeoLite2 DB as it does not exsist!") - self.logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) + self.logger.debug("Opening GeoLite2 tar file : %s", tar_dbfile) tar = taropen(tar_dbfile, 'r:gz') From da12269e732c1d35650827d75b7ec6ed65bdeaa4 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 14:16:34 -0400 Subject: [PATCH 12/20] Blank lines removed --- varken/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index ac39da4f..3ad53242 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -62,7 +62,6 @@ def update(self): dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60) - if db_next_update < today: self.logger.info("Newer GeoLite2 DB available, Updating...") self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", @@ -82,7 +81,6 @@ def update(self): self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", dbdate, db_next_update, today) - def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' From 7d657152e691ea5019b5e6acd023a1d1cf046478 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 14:18:10 -0400 Subject: [PATCH 13/20] Unused import removed --- varken/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/varken/helpers.py b/varken/helpers.py index 3ad53242..05f6978b 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -4,7 +4,6 @@ from logging import getLogger from ipaddress import IPv4Address from urllib.error import HTTPError -from calendar import monthcalendar from geoip2.database import Reader from tarfile import open as taropen from urllib3 import disable_warnings From 91c0e2148f81eb104ebea7c359fa87a5cf430842 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 16:36:44 -0400 Subject: [PATCH 14/20] Clean up logging logic for DB check --- varken/helpers.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 05f6978b..459b51b9 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -69,16 +69,10 @@ def update(self): self.download() self.reader_manager(action='open') else: - td = dbdate - today - if td.days < 0: - self.logger.debug("Geolite2 DB is only %s days old. Keeping current copy. Next update after %s", - abs(td.days), db_next_update) - self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", - dbdate, db_next_update, today) - else: - self.logger.debug("Geolite2 DB will update in %s days", abs(td.days)) - self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", - dbdate, db_next_update, today) + db_days_update = db_next_update - today + self.logger.debug("Geolite2 DB will update in %s days", abs(db_days_update.days)) + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) @@ -117,7 +111,11 @@ def download(self): tar.extract(files, self.data_folder) self.logger.debug('%s has been extracted to %s', files, self.data_folder) tar.close() - remove(tar_dbfile) + try: + remove(tar_dbfile) + self.logger.debug('Removed the GeoLite2 DB TAR file.') + except FileNotFoundError: + self.logger.warn("Cannot remove GeoLite2 DB TAR file as it does not exsist!") def hashit(string): From 620eea4ca24bec81d149555786acbf257eb319c2 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:36:33 -0400 Subject: [PATCH 15/20] Version bump --- varken/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varken/__init__.py b/varken/__init__.py index 1422b2e0..7bdf0478 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ -VERSION = "1.6.4" +VERSION = "1.6.5" BRANCH = 'master' From c7e0527930277516f2f92605b86b31c599e0c873 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:39:11 -0400 Subject: [PATCH 16/20] Relayed is an integer --- varken/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varken/structures.py b/varken/structures.py index cf3e984a..0957e446 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -456,4 +456,4 @@ class TautulliStream(NamedTuple): writers: list = None year: str = None secure: str = None - relayed: str = None + relayed: int = None From 64cc15c73a4ccf691c87b3f64b0823594513841e Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:46:50 -0400 Subject: [PATCH 17/20] Add new "relayed" and "secure" to Tautulli data pushed to influx #114 --- varken/tautulli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/varken/tautulli.py b/varken/tautulli.py index 579cd8ff..7dea8a4b 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -140,6 +140,8 @@ def get_activity(self): "longitude": longitude, "player_state": player_state, "device_type": session.platform, + "relayed": session.relayed, + "secure": session.secure, "server": self.server.id }, "time": now, From 9e4f954b3f0d9a2953db8f00778ddf1955c035da Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:54:03 -0400 Subject: [PATCH 18/20] Update README for master buiild status --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb61fe55..8f7e31f3 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@

- -[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Varken/master)](https://jenkins.cajun.pro/job/Varken/develop) + +[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Varken/master)](https://jenkins.cajun.pro/job/Varken/job/master/) [![Discord](https://img.shields.io/discord/518970285773422592.svg?colorB=7289DA&label=Discord&logo=Discord&logoColor=7289DA&style=flat-square)](https://discord.gg/VjZ6qSM) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) From 05ca5622f875358ad8a9628cb3128663307396ed Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 19:35:44 -0400 Subject: [PATCH 19/20] Update CHANGELOG --- CHANGELOG.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0756b42e..30b05038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,29 @@ # Change Log -## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-03) -[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...v1.6.4) +## [v1.6.5](https://github.com/Boerderij/Varken/tree/v1.6.5) (2019-03-11) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.6.4...v1.6.5) + +**Implemented enhancements:** + +- \[Feature Request\] Add new "relayed" and "secure" to Tautulli data pushed to influx [\#114](https://github.com/Boerderij/Varken/issues/114) +- \[BUG\] Changes to Tautulli breaks Varken `TypeError` `Secure` `relayed` [\#111](https://github.com/Boerderij/Varken/issues/111) + +**Fixed bugs:** + +- \[BUG\] Handle GeoIP Downloads better [\#113](https://github.com/Boerderij/Varken/issues/113) +- \[BUG\] - "None" outputted to stdout many times with no benefit? [\#105](https://github.com/Boerderij/Varken/issues/105) +- \[BUG\] windows file open error [\#104](https://github.com/Boerderij/Varken/issues/104) +- \[BUG\] Not catching DB url resolve [\#103](https://github.com/Boerderij/Varken/issues/103) + +**Merged pull requests:** + +- v1.6.5 Merge [\#115](https://github.com/Boerderij/Varken/pull/115) ([samwiseg0](https://github.com/samwiseg0)) + +## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-04) +[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.4...v1.6.4) + +## [1.6.4](https://github.com/Boerderij/Varken/tree/1.6.4) (2019-02-04) +[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...1.6.4) **Fixed bugs:** From eb8da9c0534c8bd92c2bd837cc2e0f1f5f31a28b Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 19:37:12 -0400 Subject: [PATCH 20/20] Remove dupe --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30b05038..93351f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,6 @@ - v1.6.5 Merge [\#115](https://github.com/Boerderij/Varken/pull/115) ([samwiseg0](https://github.com/samwiseg0)) -## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-04) -[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.4...v1.6.4) - ## [1.6.4](https://github.com/Boerderij/Varken/tree/1.6.4) (2019-02-04) [Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...1.6.4)