Merge pull request #763 from shlinkio/develop

Release 2.2.0
shlinkio · May 9, 2020 · cb6756d · cb6756d
2 parents a30f796 + cf60540
commit cb6756d
Show file tree

Hide file tree

Showing 122 changed files with 2,906 additions and 397 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -8,26 +8,25 @@ php:
   - '7.4'
 
 services:
-  - mysql
-  - postgresql
   - docker
 
 cache:
   directories:
     - $HOME/.composer/cache/files
 
 before_install:
+  - sudo ./data/infra/ci/install-ms-odbc.sh
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria
+  - yes | pecl install pdo_sqlsrv swoole-4.4.18
   - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
-  - yes | pecl install swoole-4.4.15
   - phpenv config-rm xdebug.ini || return 0
 
 install:
   - composer self-update
   - composer install --no-interaction --prefer-dist
 
 before_script:
-  - mysql -e 'CREATE DATABASE shlink_test;'
-  - psql -c 'create database shlink_test;' -U postgres
+  - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;"
   - mkdir build
   - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile)
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,58 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org).
 
+## 2.2.0 - 2020-05-09
+
+#### Added
+
+* [#712](https://github.com/shlinkio/shlink/issues/712) Added support to integrate Shlink with a [mercure hub](https://mercure.rocks/) server.
+
+    Thanks to that, Shlink will be able to publish events that can be consumed in real time.
+
+    For now, two topics (events) are published, when new visits occur. Both include a payload with the visit and the shortUrl:
+
+        * A visit occurs on any short URL: `https://shlink.io/new-visit`.
+        * A visit occurs on short URLs with a specific short code: `https://shlink.io/new-visit/{shortCode}`.
+
+    The updates are only published when serving Shlink with swoole.
+
+    Also, Shlink exposes a new endpoint `GET /rest/v2/mercure-info`, which returns the public URL of the mercure hub, and a valid JWT that can be used to subscribe to updates.
+
+* [#673](https://github.com/shlinkio/shlink/issues/673) Added new `[GET /visits]` rest endpoint which returns basic visits stats.
+* [#674](https://github.com/shlinkio/shlink/issues/674) Added new `[GET /tags/{tag}/visits]` rest endpoint which returns visits by tag.
+
+    It works in the same way as the `[GET /short-urls/{shortCode}/visits]` one, returning the same response payload, and supporting the same query params, but the response is the list of visits in all short URLs which have provided tag.
+
+* [#672](https://github.com/shlinkio/shlink/issues/672) Enhanced `[GET /tags]` rest endpoint so that it is possible to get basic stats info for every tag.
+
+    Now, if the `withStats=true` query param is provided, the response payload will include a new `stats` property which is a list with the amount of short URLs and visits for every tag.
+
+    Also, the `tag:list` CLI command has been changed and it always behaves like this.
+
+* [#640](https://github.com/shlinkio/shlink/issues/640) Allowed to optionally disable visitors' IP address anonymization. This will make Shlink no longer be GDPR-compliant, but it's OK if you only plan to share your URLs in countries without this regulation.
+
+#### Changed
+
+* [#692](https://github.com/shlinkio/shlink/issues/692) Drastically improved performance when loading visits. Specially noticeable when loading big result sets.
+* [#657](https://github.com/shlinkio/shlink/issues/657) Updated how DB tests are run in travis by using docker containers which allow all engines to be covered.
+* [#751](https://github.com/shlinkio/shlink/issues/751) Updated PHP and swoole versions used in docker image, and removed mssql-tools, as they are not needed.
+
+#### Deprecated
+
+* *Nothing*
+
+#### Removed
+
+* *Nothing*
+
+#### Fixed
+
+* [#729](https://github.com/shlinkio/shlink/issues/729) Fixed weird error when fetching multiple visits result sets concurrently using mariadb or mysql.
+* [#735](https://github.com/shlinkio/shlink/issues/735) Fixed error when cleaning metadata cache during installation when APCu is enabled.
+* [#677](https://github.com/shlinkio/shlink/issues/677) Fixed `/health` endpoint returning `503` fail responses when the database connection has expired.
+* [#732](https://github.com/shlinkio/shlink/issues/732) Fixed wrong client IP in access logs when serving app with swoole behind load balancer.
+
+
 ## 2.1.4 - 2020-04-30
 
 #### Added

diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,8 @@
-FROM php:7.4.2-alpine3.11 as base
+FROM php:7.4.5-alpine3.11 as base
 
-ARG SHLINK_VERSION=2.0.5
+ARG SHLINK_VERSION=2.1.4
 ENV SHLINK_VERSION ${SHLINK_VERSION}
-ENV SWOOLE_VERSION 4.4.15
+ENV SWOOLE_VERSION 4.4.18
 ENV LC_ALL "C"
 
 WORKDIR /etc/shlink
@@ -25,15 +25,12 @@ RUN \
 
 # Install swoole and sqlsrv driver
 RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \
-    wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.1.1-1_amd64.apk && \
     apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \
-    apk add --allow-untrusted mssql-tools_17.5.1.1-1_amd64.apk && \
     apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \
     pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv && \
     docker-php-ext-enable swoole pdo_sqlsrv && \
     apk del .phpize-deps && \
-    rm msodbcsql17_17.5.1.1-1_amd64.apk && \
-    rm mssql-tools_17.5.1.1-1_amd64.apk
+    rm msodbcsql17_17.5.1.1-1_amd64.apk
 
 
 # Install shlink

diff --git a/composer.json b/composer.json
@@ -17,13 +17,11 @@
         "ext-pdo": "*",
         "akrabat/ip-address-middleware": "^1.0",
         "cakephp/chronos": "^1.2",
-        "cocur/slugify": "^3.0",
         "doctrine/cache": "^1.9",
         "doctrine/dbal": "^2.10",
         "doctrine/migrations": "^2.2",
         "doctrine/orm": "^2.7",
         "endroid/qr-code": "^3.6",
-        "firebase/php-jwt": "^4.0",
         "geoip2/geoip2": "^2.9",
         "guzzlehttp/guzzle": "^6.5.1",
         "laminas/laminas-config": "^3.3",
@@ -34,13 +32,14 @@
         "laminas/laminas-paginator": "^2.8",
         "laminas/laminas-servicemanager": "^3.4",
         "laminas/laminas-stdlib": "^3.2",
+        "lcobucci/jwt": "^4.0@alpha",
         "lstrojny/functional-php": "^1.9",
         "mezzio/mezzio": "^3.2",
         "mezzio/mezzio-fastroute": "^3.0",
         "mezzio/mezzio-helpers": "^5.3",
         "mezzio/mezzio-platesrenderer": "^2.1",
         "mezzio/mezzio-problem-details": "^1.1",
-        "mezzio/mezzio-swoole": "^2.6",
+        "mezzio/mezzio-swoole": "^2.6.4",
         "monolog/monolog": "^2.0",
         "nikolaposa/monolog-factory": "^3.0",
         "ocramius/proxy-manager": "^2.7.0",
@@ -49,14 +48,15 @@
         "predis/predis": "^1.1",
         "pugx/shortid-php": "^0.5",
         "ramsey/uuid": "^3.9",
-        "shlinkio/shlink-common": "^3.0",
+        "shlinkio/shlink-common": "^3.1.0",
         "shlinkio/shlink-config": "^1.0",
         "shlinkio/shlink-event-dispatcher": "^1.4",
-        "shlinkio/shlink-installer": "^4.4.0",
+        "shlinkio/shlink-installer": "^5.0.0",
         "shlinkio/shlink-ip-geolocation": "^1.4",
         "symfony/console": "^5.0",
         "symfony/filesystem": "^5.0",
         "symfony/lock": "^5.0",
+        "symfony/mercure": "^0.3.0",
         "symfony/process": "^5.0"
     },
     "require-dev": {
@@ -109,7 +109,7 @@
         ],
         "test:ci": [
             "@test:unit:ci",
-            "@test:db:ci",
+            "@test:db",
             "@test:api:ci"
         ],
         "test:unit": "phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox",
@@ -121,11 +121,6 @@
             "@test:db:postgres",
             "@test:db:ms"
         ],
-        "test:db:ci": [
-            "@test:db:sqlite",
-            "@test:db:mysql",
-            "@test:db:postgres"
-        ],
         "test:db:sqlite": "APP_ENV=test phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-db.cov --testdox -c phpunit-db.xml",
         "test:db:mysql": "DB_DRIVER=mysql composer test:db:sqlite",
         "test:db:maria": "DB_DRIVER=maria composer test:db:sqlite",
@@ -152,8 +147,7 @@
         "test:ci": "<fg=blue;options=bold>Runs all test suites, generating all needed reports and logs for CI envs</>",
         "test:unit": "<fg=blue;options=bold>Runs unit test suites</>",
         "test:unit:ci": "<fg=blue;options=bold>Runs unit test suites, generating all needed reports and logs for CI envs</>",
-        "test:db": "<fg=blue;options=bold>Runs database test suites on a SQLite, MySQL, MariaDB and PostgreSQL</>",
-        "test:db:ci": "<fg=blue;options=bold>Runs database test suites on a SQLite, MySQL and PostgreSQL</>",
+        "test:db": "<fg=blue;options=bold>Runs database test suites on a SQLite, MySQL, MariaDB, PostgreSQL and MsSQL</>",
         "test:db:sqlite": "<fg=blue;options=bold>Runs database test suites on a SQLite database</>",
         "test:db:mysql": "<fg=blue;options=bold>Runs database test suites on a MySQL database</>",
         "test:db:maria": "<fg=blue;options=bold>Runs database test suites on a MariaDB database</>",

diff --git a/config/autoload/entity-manager.local.php.dist b/config/autoload/entity-manager.local.php.dist
@@ -2,6 +2,14 @@
 
 declare(strict_types=1);
 
+use function Shlinkio\Shlink\Common\env;
+
+// When running tests, any mysql-specific option can interfere with other drivers
+$driverOptions = env('APP_ENV') === 'test' ? [] : [
+    PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8',
+    PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true,
+];
+
 return [
 
     'entity_manager' => [
@@ -10,9 +18,7 @@ return [
             'password' => 'root',
             'driver' => 'pdo_mysql',
             'host' => 'shlink_db',
-            'driverOptions' => [
-                PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8',
-            ],
+            'driverOptions' => $driverOptions,
         ],
     ],
 

diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php
@@ -8,30 +8,35 @@
 
     'installer' => [
         'enabled_options' => [
-            Option\DatabaseDriverConfigOption::class,
-            Option\DatabaseNameConfigOption::class,
-            Option\DatabaseHostConfigOption::class,
-            Option\DatabasePortConfigOption::class,
-            Option\DatabaseUserConfigOption::class,
-            Option\DatabasePasswordConfigOption::class,
-            Option\DatabaseSqlitePathConfigOption::class,
-            Option\DatabaseMySqlOptionsConfigOption::class,
-            Option\ShortDomainHostConfigOption::class,
-            Option\ShortDomainSchemaConfigOption::class,
-            Option\ValidateUrlConfigOption::class,
-            Option\VisitsWebhooksConfigOption::class,
-            Option\BaseUrlRedirectConfigOption::class,
-            Option\InvalidShortUrlRedirectConfigOption::class,
-            Option\Regular404RedirectConfigOption::class,
+            Option\Database\DatabaseDriverConfigOption::class,
+            Option\Database\DatabaseNameConfigOption::class,
+            Option\Database\DatabaseHostConfigOption::class,
+            Option\Database\DatabasePortConfigOption::class,
+            Option\Database\DatabaseUserConfigOption::class,
+            Option\Database\DatabasePasswordConfigOption::class,
+            Option\Database\DatabaseSqlitePathConfigOption::class,
+            Option\Database\DatabaseMySqlOptionsConfigOption::class,
+            Option\UrlShortener\ShortDomainHostConfigOption::class,
+            Option\UrlShortener\ShortDomainSchemaConfigOption::class,
+            Option\UrlShortener\ValidateUrlConfigOption::class,
+            Option\Visit\VisitsWebhooksConfigOption::class,
+            Option\Redirect\BaseUrlRedirectConfigOption::class,
+            Option\Redirect\InvalidShortUrlRedirectConfigOption::class,
+            Option\Redirect\Regular404RedirectConfigOption::class,
             Option\DisableTrackParamConfigOption::class,
-            Option\CheckVisitsThresholdConfigOption::class,
-            Option\VisitsThresholdConfigOption::class,
+            Option\Visit\CheckVisitsThresholdConfigOption::class,
+            Option\Visit\VisitsThresholdConfigOption::class,
             Option\BasePathConfigOption::class,
-            Option\TaskWorkerNumConfigOption::class,
-            Option\WebWorkerNumConfigOption::class,
+            Option\Worker\TaskWorkerNumConfigOption::class,
+            Option\Worker\WebWorkerNumConfigOption::class,
             Option\RedisServersConfigOption::class,
-            Option\ShortCodeLengthOption::class,
-            Option\GeoLiteLicenseKeyConfigOption::class,
+            Option\UrlShortener\ShortCodeLengthOption::class,
+            Option\Mercure\EnableMercureConfigOption::class,
+            Option\Mercure\MercurePublicUrlConfigOption::class,
+            Option\Mercure\MercureInternalUrlConfigOption::class,
+            Option\Mercure\MercureJwtSecretConfigOption::class,
+            Option\UrlShortener\GeoLiteLicenseKeyConfigOption::class,
+            Option\UrlShortener\IpAnonymizationConfigOption::class,
         ],
 
         'installation_commands' => [

diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php
@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+use Laminas\ServiceManager\Proxy\LazyServiceFactory;
+use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider;
+use Symfony\Component\Mercure\Publisher;
+use Symfony\Component\Mercure\PublisherInterface;
+
+return [
+
+    'mercure' => [
+        'public_hub_url' => null,
+        'internal_hub_url' => null,
+        'jwt_secret' => null,
+        'jwt_issuer' => 'Shlink',
+    ],
+
+    'dependencies' => [
+        'delegators' => [
+            LcobucciJwtProvider::class => [
+                LazyServiceFactory::class,
+            ],
+            Publisher::class => [
+                LazyServiceFactory::class,
+            ],
+        ],
+        'lazy_services' => [
+            'class_map' => [
+                LcobucciJwtProvider::class => LcobucciJwtProvider::class,
+                Publisher::class => PublisherInterface::class,
+            ],
+        ],
+    ],
+
+];
diff --git a/config/autoload/mercure.local.php.dist b/config/autoload/mercure.local.php.dist
@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+return [
+
+    'mercure' => [
+        'public_hub_url' => 'http://localhost:8001',
+        'internal_hub_url' => 'http://shlink_mercure_proxy',
+        'jwt_secret' => 'mercure_jwt_key',
+    ],
+
+];
diff --git a/config/autoload/swoole.global.php b/config/autoload/swoole.global.php
@@ -5,7 +5,8 @@
 return [
 
     'mezzio-swoole' => [
-        'enable_coroutine' => true,
+        // Setting this to true can have unexpected behaviors when running several concurrent slow DB queries
+        'enable_coroutine' => false,
 
         'swoole-http-server' => [
             'host' => '0.0.0.0',

diff --git a/config/autoload/url-shortener.global.php b/config/autoload/url-shortener.global.php
@@ -12,6 +12,7 @@
             'hostname' => '',
         ],
         'validate_url' => false,
+        'anonymize_remote_addr' => true,
         'visits_webhooks' => [],
         'default_short_codes_length' => DEFAULT_SHORT_CODES_LENGTH,
     ],

diff --git a/config/config.php b/config/config.php
@@ -5,7 +5,6 @@
 namespace Shlinkio\Shlink;
 
 use Laminas\ConfigAggregator;
-use Laminas\ZendFrameworkBridge;
 use Mezzio;
 use Mezzio\ProblemDetails;
 
@@ -30,7 +29,6 @@
         ? new ConfigAggregator\PhpFileProvider('config/test/*.global.php')
         : new ConfigAggregator\LaminasConfigProvider('config/params/{generated_config.php,*.config.{php,json}}'),
 ], 'data/cache/app_config.php', [
-    ZendFrameworkBridge\ConfigPostProcessor::class,
     Core\Config\SimplifiedConfigParser::class,
     Core\Config\BasePathPrefixer::class,
     Core\Config\DeprecatedConfigParser::class,