diff --git a/.travis.yml b/.travis.yml index 6ab12fa8c..9b1c35e69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,25 @@ +dist: bionic + language: php branches: only: - /.*/ -php: - - '7.4' +jobs: + fast_finish: true + include: + - name: "Docker publish" + php: '7.4' + if: NOT type = pull_request + env: + - DOCKER_PUBLISH="true" + - name: "CI" + php: '7.4' + env: + - DOCKER_PUBLISH="false" + allow_failures: + - name: "Docker publish" services: - docker @@ -15,36 +29,38 @@ cache: - $HOME/.composer/cache/files before_install: - - sudo ./data/infra/ci/install-ms-odbc.sh - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - - yes | pecl install pdo_sqlsrv swoole-4.4.18 - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then sudo ./data/infra/ci/install-ms-odbc.sh ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then yes | pecl install pdo_sqlsrv swoole-4.4.18 ; fi install: - - composer self-update - - composer install --no-interaction --prefer-dist + - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then sudo ./data/infra/ci/install-docker.sh ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer self-update ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer install --no-interaction --prefer-dist ; fi before_script: - - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" ; fi - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) script: - - composer ci - - if [[ ! -z "$DOCKERFILE_CHANGED" && "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then docker build -t shlink-docker-image:temp . ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then bin/test/run-api-tests.sh --coverage-php build/coverage-api.cov && composer ci ; fi + - if [[ ! -z "${DOCKERFILE_CHANGED}" && "${TRAVIS_PHP_VERSION}" == "7.4" && "${DOCKER_PUBLISH}" == "false" ]]; then docker build -t shlink-docker-image:temp . ; fi + - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then bash ./docker/build ; fi after_success: - rm -f build/clover.xml - - wget https://phar.phpunit.de/phpcov-7.0.2.phar - - phpdbg -qrr phpcov-7.0.2.phar merge build --clover build/clover.xml - - wget https://scrutinizer-ci.com/ocular.phar - - php ocular.phar code-coverage:upload --format=php-clover build/clover.xml + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then wget https://phar.phpunit.de/phpcov-7.0.2.phar ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then phpdbg -qrr phpcov-7.0.2.phar merge build --clover build/clover.xml ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then wget https://scrutinizer-ci.com/ocular.phar ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then php ocular.phar code-coverage:upload --format=php-clover build/clover.xml ; fi # Before deploying, build dist file for current travis tag before_deploy: - rm -f ocular.phar - - if [[ ! -z $TRAVIS_TAG && "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then ./build.sh ${TRAVIS_TAG#?} ; fi + - if [[ ! -z ${TRAVIS_TAG} && "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then ./build.sh ${TRAVIS_TAG#?} ; fi deploy: - provider: releases @@ -52,12 +68,8 @@ deploy: secure: a9dbZchocqeuOViwUeNH54bQR5Sz7rEYXx5b9WPFtnFn9LGKKUaLbA2U91UQ9QKPrcTpsALubUYbw2CnNmvCwzaY+R8lCD3gkU4ohsEnbpnw3deOeixI74sqBHJAuCH9FSaRDGILoBMtUKx2xlzIymFxkIsgIukkGbdkWHDlRWY3oTUUuw1SQ2Xk9KDsbJQtjIc1+G/O6gHaV4qv/R9W8NPmJExKTNDrAZbC1vIUnxqp4UpVo1hst8qPd1at94CndDYM5rG+7imGbdtxTxzamt819qdTO1OfvtctKawNAm7YXZrrWft6c7gI6j6SI4hxd+ZrrPBqbaRFHkZHjnNssO/yn4SaOHFFzccmu0MzvpPCf0qWZwd3sGHVYer1MnR2mHYqU84QPlW3nrHwJjkrpq3+q0JcBY6GsJs+RskHNtkMTKV05Iz6QUI5YZGwTpuXaRm036SmavjGc4IDlMaYCk/NmbB9BKpthJxLdUpczOHpnjXXHziotWD6cfEnbjU3byfD8HY5WrxSjsNT7SKmXN3hRof7bk985ewQVjGT42O3NbnfnqjQQWr/B7/zFTpLR4f526Bkq12CdCyf5lvrbq+POkLVdJ+uFfR7ds248Ue/jBQy6kM1tWmKF9QiwisFlA84eQ4CW3I93Rp97URv+AQa9zmbD0Ve3Udp+g6nF5I= file: "./build/shlink_${TRAVIS_TAG#?}_dist.zip" skip_cleanup: true - on: - tags: true - php: '7.4' - - provider: script - script: bash ./docker/build on: all_branches: true - condition: $TRAVIS_PULL_REQUEST == 'false' + condition: ${DOCKER_PUBLISH} == 'false' + tags: true php: '7.4' diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ede91847..6192481f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). +## 2.2.2 - 2020-06-08 + +#### Added + +* [#709](https://github.com/shlinkio/shlink/issues/709) Added multi-architecture builds for the docker image. + +#### Changed + +* *Nothing* + +#### Deprecated + +* *Nothing* + +#### Removed + +* *Nothing* + +#### Fixed + +* [#769](https://github.com/shlinkio/shlink/issues/769) Fixed custom slugs not allowing valid URL characters, like `.`, `_` or `~`. +* [#781](https://github.com/shlinkio/shlink/issues/781) Fixed memory leak when loading visits for a tag which is used for big amounts of short URLs. + + ## 2.2.1 - 2020-05-11 #### Added diff --git a/Dockerfile b/Dockerfile index 3f2c5856c..75c4ae2d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,14 +23,22 @@ RUN \ apk add --no-cache libzip-dev zlib-dev libpng-dev && \ docker-php-ext-install -j"$(nproc)" zip gd -# Install swoole and sqlsrv driver -RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ - pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv && \ - docker-php-ext-enable swoole pdo_sqlsrv && \ - apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk +# Install sqlsrv driver +RUN if [ $(uname -m) == "x86_64" ]; then \ + wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ + apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ + apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ + pecl install pdo_sqlsrv && \ + docker-php-ext-enable pdo_sqlsrv && \ + apk del .phpize-deps && \ + rm msodbcsql17_17.5.1.1-1_amd64.apk ; \ + fi + +# Install swoole +RUN apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS && \ + pecl install swoole-${SWOOLE_VERSION} && \ + docker-php-ext-enable swoole && \ + apk del .phpize-deps # Install shlink diff --git a/composer.json b/composer.json index 5d531a370..ef9e0e928 100644 --- a/composer.json +++ b/composer.json @@ -17,6 +17,7 @@ "ext-pdo": "*", "akrabat/ip-address-middleware": "^1.0", "cakephp/chronos": "^1.2", + "cocur/slugify": "^4.0", "doctrine/cache": "^1.9", "doctrine/dbal": "^2.10", "doctrine/migrations": "^2.2", @@ -53,11 +54,13 @@ "shlinkio/shlink-event-dispatcher": "^1.4", "shlinkio/shlink-installer": "^5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", - "symfony/console": "^5.0", - "symfony/filesystem": "^5.0", - "symfony/lock": "^5.0", + "symfony/console": "^5.1", + "symfony/filesystem": "^5.1", + "symfony/lock": "^5.1", "symfony/mercure": "^0.3.0", - "symfony/process": "^5.0" + "symfony/process": "^5.1", + "symfony/string": "^5.1", + "symfony/translation-contracts": "^2.1" }, "require-dev": { "devster/ubench": "^2.0", @@ -109,8 +112,7 @@ ], "test:ci": [ "@test:unit:ci", - "@test:db", - "@test:api:ci" + "@test:db" ], "test:unit": "phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", "test:unit:ci": "@test:unit --coverage-clover=build/clover.xml --coverage-xml=build/coverage-xml --log-junit=build/junit.xml", diff --git a/data/infra/ci/install-docker.sh b/data/infra/ci/install-docker.sh new file mode 100755 index 000000000..58289a16b --- /dev/null +++ b/data/infra/ci/install-docker.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -ex + +# install latest docker version +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + +# enable multiarch execution +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes diff --git a/docker/README.md b/docker/README.md index e17e570e3..199dc6e24 100644 --- a/docker/README.md +++ b/docker/README.md @@ -263,7 +263,13 @@ Once created just run shlink with the volume: docker run --name shlink -p 8080:8080 -v ${PWD}/my/config/dir:/etc/shlink/config/params shlinkio/shlink:stable ``` -## Multi instance considerations +## Multi-architecture + +Starting on v2.3.0, Shlink's docker image is built for multiple architectures. + +The only limitation is that images for architectures other than `amd64` will not have support for Microsoft SQL databases, since there are no official binaries. + +## Multi-instance considerations These are some considerations to take into account when running multiple instances of shlink. diff --git a/docker/build b/docker/build index f7e4b9231..8ac27d4d2 100755 --- a/docker/build +++ b/docker/build @@ -1,17 +1,35 @@ #!/bin/bash + set -e +# PLATFORMS="linux/arm/v7,linux/arm64/v8,linux/amd64" +PLATFORMS="linux/amd64" +DOCKER_IMAGE="shlinkio/shlink" +BUILDX_VER=v0.4.1 +export DOCKER_CLI_EXPERIMENTAL=enabled + +mkdir -vp ~/.docker/cli-plugins/ ~/dockercache +curl --silent -L "https://github.com/docker/buildx/releases/download/${BUILDX_VER}/buildx-${BUILDX_VER}.linux-amd64" > ~/.docker/cli-plugins/docker-buildx +chmod a+x ~/.docker/cli-plugins/docker-buildx + +docker buildx create --use + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # If there is a tag, regardless the branch, build that docker tag and also "stable" if [[ ! -z $TRAVIS_TAG ]]; then - docker build --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} -t shlinkio/shlink:${TRAVIS_TAG#?} -t shlinkio/shlink:stable . - docker push shlinkio/shlink:${TRAVIS_TAG#?} - + TAGS="-t ${DOCKER_IMAGE}:${TRAVIS_TAG#?}" # Push stable tag only if this is not an alpha or beta tag - [[ $TRAVIS_TAG != *"alpha"* && $TRAVIS_TAG != *"beta"* ]] && docker push shlinkio/shlink:stable + [[ $TRAVIS_TAG != *"alpha"* && $TRAVIS_TAG != *"beta"* ]] && TAGS="${TAGS} -t ${DOCKER_IMAGE}:stable" + + docker buildx build --push \ + --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ + --platform ${PLATFORMS} \ + ${TAGS} . + # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then - docker build -t shlinkio/shlink:latest . - docker push shlinkio/shlink:latest + docker buildx build --push \ + --platform ${PLATFORMS} \ + -t ${DOCKER_IMAGE}:latest . fi diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index b3761c9f9..458b8ef2e 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -12,8 +12,6 @@ use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; -use function array_column; - use const PHP_INT_MAX; class VisitRepository extends EntityRepository implements VisitRepositoryInterface @@ -142,26 +140,18 @@ public function countVisitsByTag(string $tag, ?DateRange $dateRange = null): int private function createVisitsByTagQueryBuilder(string $tag, ?DateRange $dateRange = null): QueryBuilder { - $qb = $this->getEntityManager()->createQueryBuilder(); - $qb->select('s.id') - ->from(ShortUrl::class, 's') - ->join('s.tags', 't') - ->where($qb->expr()->eq('t.name', ':tag')) - ->setParameter('tag', $tag); - - $shortUrlIds = array_column($qb->getQuery()->getArrayResult(), 'id'); - $shortUrlIds[] = '-1'; // Add an invalid ID, in case the list is empty - // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later // Since they are not strictly provided by the caller, it's reasonably safe - $qb2 = $this->getEntityManager()->createQueryBuilder(); - $qb2->from(Visit::class, 'v') - ->where($qb2->expr()->in('v.shortUrl', $shortUrlIds)); + $qb = $this->getEntityManager()->createQueryBuilder(); + $qb->from(Visit::class, 'v') + ->join('v.shortUrl', 's') + ->join('s.tags', 't') + ->where($qb->expr()->eq('t.name', '\'' . $tag . '\'')); // Apply date range filtering - $this->applyDatesInline($qb2, $dateRange); + $this->applyDatesInline($qb, $dateRange); - return $qb2; + return $qb; } private function applyDatesInline(QueryBuilder $qb, ?DateRange $dateRange): void diff --git a/module/Core/src/Util/CocurSymfonySluggerBridge.php b/module/Core/src/Util/CocurSymfonySluggerBridge.php new file mode 100644 index 000000000..9415e47c6 --- /dev/null +++ b/module/Core/src/Util/CocurSymfonySluggerBridge.php @@ -0,0 +1,26 @@ +slugger = $slugger; + } + + public function slug(string $string, string $separator = '-', ?string $locale = null): AbstractUnicodeString + { + return s($this->slugger->slugify($string, $separator)); + } +} diff --git a/module/Core/src/Validation/ShortUrlMetaInputFilter.php b/module/Core/src/Validation/ShortUrlMetaInputFilter.php index 8fde5e98c..4503117c5 100644 --- a/module/Core/src/Validation/ShortUrlMetaInputFilter.php +++ b/module/Core/src/Validation/ShortUrlMetaInputFilter.php @@ -4,11 +4,13 @@ namespace Shlinkio\Shlink\Core\Validation; +use Cocur\Slugify\Slugify; use DateTime; use Laminas\InputFilter\Input; use Laminas\InputFilter\InputFilter; use Laminas\Validator; use Shlinkio\Shlink\Common\Validation; +use Shlinkio\Shlink\Core\Util\CocurSymfonySluggerBridge; use const Shlinkio\Shlink\Core\MIN_SHORT_CODES_LENGTH; @@ -46,7 +48,10 @@ private function initialize(): void // FIXME The only way to enforce the NotEmpty validator to be evaluated when the value is provided but it's // empty, is by using the deprecated setContinueIfEmpty $customSlug = $this->createInput(self::CUSTOM_SLUG, false)->setContinueIfEmpty(true); - $customSlug->getFilterChain()->attach(new Validation\SluggerFilter()); + $customSlug->getFilterChain()->attach(new Validation\SluggerFilter(new CocurSymfonySluggerBridge(new Slugify([ + 'regexp' => '/[^A-Za-z0-9._~]+/', + 'lowercase' => false, + ])))); $customSlug->getValidatorChain()->attach(new Validator\NotEmpty([ Validator\NotEmpty::STRING, Validator\NotEmpty::SPACE, diff --git a/module/Core/test/Model/ShortUrlMetaTest.php b/module/Core/test/Model/ShortUrlMetaTest.php index 7d0dd9b61..fe3d42fc8 100644 --- a/module/Core/test/Model/ShortUrlMetaTest.php +++ b/module/Core/test/Model/ShortUrlMetaTest.php @@ -58,11 +58,14 @@ public function provideInvalidData(): iterable ]]; } - /** @test */ - public function properlyCreatedInstanceReturnsValues(): void + /** + * @test + * @dataProvider provideCustomSlugs + */ + public function properlyCreatedInstanceReturnsValues(string $customSlug, string $expectedSlug): void { $meta = ShortUrlMeta::fromRawData( - ['validSince' => Chronos::parse('2015-01-01')->toAtomString(), 'customSlug' => 'foobar'], + ['validSince' => Chronos::parse('2015-01-01')->toAtomString(), 'customSlug' => $customSlug], ); $this->assertTrue($meta->hasValidSince()); @@ -72,9 +75,18 @@ public function properlyCreatedInstanceReturnsValues(): void $this->assertNull($meta->getValidUntil()); $this->assertTrue($meta->hasCustomSlug()); - $this->assertEquals('foobar', $meta->getCustomSlug()); + $this->assertEquals($expectedSlug, $meta->getCustomSlug()); $this->assertFalse($meta->hasMaxVisits()); $this->assertNull($meta->getMaxVisits()); } + + public function provideCustomSlugs(): iterable + { + yield ['foobar', 'foobar']; + yield ['foo bar', 'foo-bar']; + yield ['wp-admin.php', 'wp-admin.php']; + yield ['UPPER_lower', 'UPPER_lower']; + yield ['more~url_special.chars', 'more~url_special.chars']; + } }