diff --git a/.gitignore b/.gitignore index 1a8c00f..6430369 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ Homestead.json Homestead.yaml npm-debug.log yarn-error.log +docker-compose.override.yml diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c19df4b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM php:7-alpine -MAINTAINER Shaark contributors - -WORKDIR /app -COPY . /app - -RUN apk add --no-cache --update openssl zip unzip oniguruma-dev zlib-dev libpng-dev libzip-dev postgresql-dev && \ - curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer && \ - docker-php-ext-install pdo mbstring gd exif zip sockets pdo_mysql pgsql pdo_pgsql && \ - cp .env.example .env && \ - \ - sed -i s/DB_HOST=127.0.0.1/DB_HOST=mariadb/ .env && \ - sed -i s/REDIS_HOST=127.0.0.1/REDIS_HOST=redis/ .env && \ - sed -i s/APP_ENV=local/APP_ENV=production/ .env && \ - sed -i s/APP_DEBUG=true/APP_DEBUG=false/ .env && \ - sed -i s/CACHE_DRIVER=file/CACHE_DRIVER=redis/ .env && \ - sed -i s/QUEUE_CONNECTION=sync/QUEUE_CONNECTION=redis/ .env && \ - sed -i s/SESSION_DRIVER=file/SESSION_DRIVER=redis/ .env && \ - sed -i s/REDIS_HOST=127.0.0.1/REDIS_HOST=redis/ .env && \ - \ - composer install --no-dev -o && \ - php artisan optimize && \ - php artisan view:clear && \ - \ - php artisan key:generate && \ - php artisan storage:link && \ - php artisan config:cache && \ - php artisan migrate --seed - -CMD php artisan serve --host=0.0.0.0 --port=80 -EXPOSE 80 diff --git a/Dockerfile.shaark b/Dockerfile.shaark new file mode 100644 index 0000000..c6b7012 --- /dev/null +++ b/Dockerfile.shaark @@ -0,0 +1,111 @@ +FROM php:7.4.0-alpine +MAINTAINER Shaark contributors + +WORKDIR /app +COPY . /app + +# Install packages needed for shaark +RUN apk add --no-cache \ + bash \ + openssl \ + zip \ + unzip \ + oniguruma-dev \ + zlib-dev \ + libpng-dev \ + libzip-dev \ + postgresql-dev \ + gmp \ + gmp-dev \ + python3 \ + git \ + libcap \ + mariadb-client \ + nodejs \ + npm \ + busybox-suid + +# Set inheritied capabilities on entrypoint +RUN setcap cap_net_raw+eip /app/app/entrypoint-shaark.sh && \ + setcap cap_sys_admin+eip /app/app/entrypoint-shaark.sh && \ + setcap cap_net_bind_service=+ep `which php` + +# Installs latest Chromium (83) package. +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + freetype-dev \ + harfbuzz \ + ca-certificates \ + ttf-freefont + +# Set environment variables +ENV \ + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser \ + DB_HOST="mariadb" \ + REDIS_HOST="redis" \ + APP_ENV="production" \ + APP_DEBUG="false" \ + APP_MIGRATE_DB="true" \ + CACHE_DRIVER="redis" \ + QUEUE_CONNECTION="redis" \ + SESSION_DRIVER="redis" \ + REDIS_HOST="redis" + +# Puppeteer v3.1.0 works with Chromium 83. +RUN npm install puppeteer@3.1.0 + +# Add user so we don't have to run everything as root +RUN addgroup -S shaark && adduser -S -G shaark shaarkuser \ + && mkdir -p /home/shaarkuser/Downloads \ + && chown -R shaarkuser:shaark /home/shaarkuser \ + && chown -R shaarkuser:shaark /app + +# Install youtube-dl binary +RUN curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/bin/youtube-dl && \ + chmod a+rx /usr/bin/youtube-dl + +# Make sure python binary is python3 +RUN if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi + +# Install composer +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && \ + php -r "if (hash_file('sha384', 'composer-setup.php') === '906a84df04cea2aa72f40b5f787e49f22d4c2f19492ac310e8cba5b96ac8b64115ac402c8cd292b8a03482574915d1a8') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" && \ + php composer-setup.php && \ + php -r "unlink('composer-setup.php');" && \ + mv composer.phar /usr/local/bin/composer + +# Install php extensions +RUN docker-php-ext-install \ + pdo \ + mbstring \ + gd \ + exif \ + zip \ + sockets \ + pdo_mysql \ + pgsql \ + pdo_pgsql \ + gmp \ + bcmath + +# Configure Backups cron +RUN crontab -u shaarkuser app/crontab + +# Run everything after as non-privileged user. +USER shaarkuser + +RUN composer install --no-dev -o + +RUN cp .env.example .env && \ + \ + php artisan optimize && \ + php artisan view:clear && \ + \ + php artisan key:generate && \ + php artisan storage:link + +EXPOSE 80 +ENTRYPOINT [ "app/entrypoint-shaark.sh" ] diff --git a/app/Http/Controllers/Api/Manage/FeaturesController.php b/app/Http/Controllers/Api/Manage/FeaturesController.php index df6548f..327c673 100644 --- a/app/Http/Controllers/Api/Manage/FeaturesController.php +++ b/app/Http/Controllers/Api/Manage/FeaturesController.php @@ -59,14 +59,14 @@ protected function checkArchivePdf(Shaark $shaark) return $this->sendError(__('Your node path is unreachable: :path', ['path' => $exec])); } - $dir = base_path('node_modules/puppeteer/.local-chromium'); + $dir = base_path('vendor/spatie/browsershot'); if (false === is_dir($dir)) { - return $this->sendError(__('Puppeteer dependencies not installed, run `npm install @nesk/puphpeteer --no-save`')); + return $this->sendError(__('Puppeteer dependencies not installed, run `composer require spatie/browsershot`')); } try { - $name = LinkArchive::archive(url()->route('home'), 'pdf'); + $name = LinkArchive::archive('http://example.com', 'pdf'); } catch (\Exception $e) { return $this->sendError(__('Unable to create archive, error is: :message', ['message' => $e->getMessage()])); } diff --git a/app/Services/LinkArchive/BrowsershotProvider.php b/app/Services/LinkArchive/BrowsershotProvider.php new file mode 100644 index 0000000..54c6731 --- /dev/null +++ b/app/Services/LinkArchive/BrowsershotProvider.php @@ -0,0 +1,53 @@ +url) . '.pdf'; + $filename = sprintf('app/archives/%s', $name); + $windowWidth = app('shaark')->getArchivePdfWidth(); + $windowHeight = app('shaark')->getArchivePdfHeight(); + $nodeBin = app('shaark')->getNodeBin(); + + try { + $browsershot = new Browsershot($this->url, true); + $browsershot + ->windowSize($windowWidth, $windowHeight) + ->margins(0,0,0,0) + ->setNodeBinary($nodeBin) + ->setNodeModulePath('node_modules/') + ->setIncludePath('/usr/bin/') + ->showBackground() + ->addChromiumArguments([ + 'disable-dev-shm-usage' + ]) + ->noSandbox() + ->ignoreHttpsErrors() + ->dismissDialogs() + ->waitUntilNetworkIdle() + ->emulateMedia('screen') + ->save(storage_path($filename)) + ; + } catch (\Exception $e) { + throw new \RuntimeException("Unable to create link archive", 0, $e); + } + + return $name; + } + + public function isEnabled(): bool + { + return app('shaark')->getLinkArchivePdf() === true; + } + + public function canArchive(): bool + { + return true; + } +} + diff --git a/app/Services/LinkArchive/LinkArchive.php b/app/Services/LinkArchive/LinkArchive.php index da39a43..303be1f 100644 --- a/app/Services/LinkArchive/LinkArchive.php +++ b/app/Services/LinkArchive/LinkArchive.php @@ -7,7 +7,7 @@ class LinkArchive /** @var array $providers */ public static $providers = [ 'media' => YoutubeDlProvider::class, - 'pdf' => PuppeteerProvider::class, + 'pdf' => BrowsershotProvider::class, ]; public static function availableFor(string $url): array diff --git a/app/Services/LinkArchive/PuppeteerProvider.php b/app/Services/LinkArchive/PuppeteerProvider.php deleted file mode 100644 index 548f673..0000000 --- a/app/Services/LinkArchive/PuppeteerProvider.php +++ /dev/null @@ -1,58 +0,0 @@ -url) . '.pdf'; - $filename = sprintf('app/archives/%s', $name); - - try { - $puppeteer = new Puppeteer([ - 'executable_path' => app('shaark')->getNodeBin() - ]); - - $browser = $puppeteer->launch([ - 'ignoreHTTPSErrors' => true, - ]); - - $page = $browser->newPage(); - $page->goto($this->url); - $page->emulateMedia('screen'); - - $page->pdf([ - 'path' => storage_path($filename), - 'width' => app('shaark')->getArchivePdfWidth(), - 'height' => app('shaark')->getArchivePdfHeight(), - 'printBackground' => true, - 'preferCSSPageSize' => true, - 'margin' => [ - 'top' => 0, - 'bottom' => 0, - 'left' => 0, - 'right' => 0, - ] - ]); - - $browser->close(); - } catch (\Exception $e) { - throw new \RuntimeException("Unable to create link pdf archive", 0, $e); - } - - return $name; - } - - public function isEnabled(): bool - { - return app('shaark')->getLinkArchivePdf() === true; - } - - public function canArchive(): bool - { - return true; - } -} diff --git a/app/crontab b/app/crontab new file mode 100644 index 0000000..eb03dcc --- /dev/null +++ b/app/crontab @@ -0,0 +1,10 @@ +# do daily/weekly/monthly maintenance +# min hour day month weekday command +*/15 * * * * run-parts /etc/periodic/15min +0 * * * * run-parts /etc/periodic/hourly +0 2 * * * run-parts /etc/periodic/daily +0 3 * * 6 run-parts /etc/periodic/weekly +0 5 1 * * run-parts /etc/periodic/monthly +# run backups configured by Shaark +* * * * * cd /app && php artisan schedule:run >> /dev/null 2>&1 + diff --git a/app/entrypoint-shaark.sh b/app/entrypoint-shaark.sh new file mode 100755 index 0000000..4f97a0f --- /dev/null +++ b/app/entrypoint-shaark.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +cd /app +echo "Clearing any cached config." +php artisan config:clear +if [ "`php artisan migrate:status`" = "Migration table not found." ]; then + echo "Migrating database and creating default Admin user." + php artisan migrate --seed --force + echo "Admin Username: admin@example.com" + echo "Admin Password: "${APP_ADMIN_PASSWORD} +elif [ "${APP_MIGRATE_DB}" = 'true' ] && \ + [ `php artisan migrate:status|cut -d'|' -f2 |grep -c "No"` -gt 0 ]; then + echo "Migrating database." + php artisan migrate --force +else + echo "Database migration skipped." +fi + +if [ "${APP_DEBUG}" = 'true' ]; then + echo "Debugging enabled: creating verbose logs at /app/storage/logs/" + php artisan queue:work >> storage/logs/artisan_queue.log & + php artisan serve --host=0.0.0.0 --port=80 -vvv >> storage/logs/artisan_serve.log +else + echo "Starting Shaark!" + php artisan queue:work & + php artisan serve --host=0.0.0.0 --port=80 +fi diff --git a/app/setEnv.sh b/app/setEnv.sh new file mode 100755 index 0000000..d3320d7 --- /dev/null +++ b/app/setEnv.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# docker-compose.*.yml files names and their position compared to this script. +# Here: in parent directory. +target_dir="${0%/*}/.." +override_link='docker-compose.override.yml' +override_file_dev='docker-compose.override.dev.yml' + + +# Get the current environment and tells what are the options +function show_current { + get_current + say_switch +} + + +# Get the current environment +# Output variable: current_env +function get_current { + if [ -L ${override_link} ] + then + # Check for Mac OSX + if [[ "$OSTYPE" == "darwin"* ]]; then + # readlink is not native to mac, so this will work in it's place. + symlink=$(python3 -c "import os; print(os.path.realpath('docker-compose.override.yml'))") + else + # Maintain the cleaner way + symlink=$(readlink -f docker-compose.override.yml) + fi + current_env=$(expr $(basename symlink) : "^docker-compose.override.\(.*\).yml$") + else + current_env=production + fi +} + +# Tell to which environments we can switch +function say_switch { + echo "Using '${current_env}' configuration." + for one_env in dev production + do + if [ "${current_env}" != ${one_env} ]; then + echo "-> You can switch to '${one_env}' with '${0} ${one_env}'" + fi + done +} + + +function set_production { + get_current + if [ "${current_env}" != production ] + then + # In production configuration there is no override file + rm ${override_link} + docker-compose down + echo "Now using 'production' configuration." + else + echo "Already using 'production' configuration." + fi +} + + +function set_dev { + get_current + if [ "${current_env}" != dev ] + then + rm -f ${override_link} + ln -s ${override_file_dev} ${override_link} + docker-compose down + echo "Now using 'dev' configuration." + else + echo "Already using 'dev' configuration." + fi +} + +# Change directory to allow working with relative paths. +cd ${target_dir} + +if [ ${#} -eq 1 ] && [[ 'dev production' =~ "${1}" ]] +then + set_"${1}" +else + show_current +fi diff --git a/app/wait-for-it.sh b/app/wait-for-it.sh new file mode 100755 index 0000000..875d2bc --- /dev/null +++ b/app/wait-for-it.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI="$@" + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-60} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec $CLI +else + exit $RESULT +fi diff --git a/composer.json b/composer.json index 79e96f1..dfb2c2d 100644 --- a/composer.json +++ b/composer.json @@ -34,10 +34,10 @@ "laravel/scout": "^7.2", "laravel/tinker": "^1.0", "maatwebsite/excel": "^3.1", - "mews/captcha": "^3.2", - "nesk/puphpeteer": "^1.6", + "mews/captcha": "^3.0", "norkunas/youtube-dl-php": "^1.6", "predis/predis": "^1.1", + "spatie/browsershot": "^3.37", "spatie/laravel-backup": "^6.11", "spatie/laravel-medialibrary": "^7.19", "spatie/valuestore": "^1.2", diff --git a/database/seeds/DatabaseSeeder.php b/database/seeds/DatabaseSeeder.php index 686b142..68ce64a 100644 --- a/database/seeds/DatabaseSeeder.php +++ b/database/seeds/DatabaseSeeder.php @@ -17,8 +17,8 @@ public function run() DB::table('users')->insert([ 'name' => 'Admin', - 'email' => 'shaark@example.com', - 'password' => Hash::make('secret'), + 'email' => 'admin@example.com', + 'password' => Hash::make(env('APP_ADMIN_PASSWORD', 'secret')), 'api_token' => 'api-token-secret', 'is_admin' => 1, 'created_at' => now()->toDateTimeString(), diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml new file mode 100644 index 0000000..67fec57 --- /dev/null +++ b/docker-compose.override.dev.yml @@ -0,0 +1,15 @@ +version: '3.7' +services: + + shaark: + environment: + APP_ENV: "dev" + APP_DEBUG: "true" + APP_MIGRATE_DB: ${SHAARK_MIGRATE_DB:-true} + + mariadb: + ports: + - target: 3306 + published: 3306 + protocol: tcp + mode: host diff --git a/docker-compose.yml b/docker-compose.yml index cc490de..c354f3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,60 +1,65 @@ -version: '2.2' +version: '3.7' services: shaark: - image: shaark - build: . - container_name: shaark + image: shaark:${SHAARK_IMAGE_TAG:-latest} + build: + context: ./ + dockerfile: Dockerfile.shaark restart: unless-stopped -# volumes: -# - "./env:/app/.env" -# ports: -# - "80:80/tcp" - networks: - - shaark - - nginx_net + ports: + - target: 80 + published: ${SHAARK_PORT:-8080} + protocol: tcp + mode: host + depends_on: + - mariadb + - redis + entrypoint: ['./app/wait-for-it.sh', 'mariadb:3306', '-t', '30', '--', './app/entrypoint-shaark.sh'] + environment: + APP_MIGRATE_DB: ${SHAARK_MIGRATE_DB:-false} + APP_ENV: "production" + APP_DEBUG: ${SHAARK_DEBUG:-false} + APP_URL: ${SHAARK_URL:-http://localhost} + APP_ADMIN_PASSWORD: ${SHAARK_ADMIN_PASSWORD:-secret} + DB_PASSWORD: ${SHAARK_DATABASE_PASSWORD:-secret} + DB_USER: ${SHAARK_DATABASE_USER:-homestead} + DB_DATABASE: ${SHAARK_DATABASE_NAME:-homestead} + volumes: + - storage:/app/storage logging: driver: "json-file" options: max-size: "10m" max-file: "5" - - mariadb: + + mariadb: image: mariadb - container_name: mariadb_shaark restart: unless-stopped volumes: - - /opt/shaark/mariadb:/var/lib/mysql - networks: - - shaark + - mariadb:/var/lib/mysql environment: - MYSQL_ROOT_PASSWORD: rootpassword9867ow3q459087w980 - MYSQL_PASSWORD: secret - MYSQL_USER: homestead - MYSQL_DATABASE: homestead + MYSQL_RANDOM_ROOT_PASSWORD: 'true' + MYSQL_PASSWORD: ${SHAARK_DATABASE_PASSWORD:-secret} + MYSQL_USER: ${SHAARK_DATABASE_USER:-homestead} + MYSQL_DATABASE: ${SHAARK_DATABASE_NAME:-homestead} logging: driver: "json-file" options: max-size: "10m" max-file: "5" - redis: image: redis - container_name: redis_shaark restart: unless-stopped volumes: - - /opt/shaark/redis:/data - networks: - - shaark + - redis:/data logging: driver: "json-file" options: max-size: "10m" max-file: "5" - -networks: - shaark: - name: shaark_net - nginx_net: - name: nginx_net +volumes: + redis: + mariadb: + storage: