diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cfb2153 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM node:10-alpine + +RUN apk add --no-cache \ + python2 \ + make \ + g++ + +RUN mkdir /app + +COPY ./app /app + +RUN cd /app && \ + npm i + +COPY ./init.sh / + +RUN chmod +x /init.sh + +COPY ./init-update.sh / + +RUN chmod +x /init-update.sh + +COPY ./feed-cron /etc/periodic/15min/ + +RUN chmod +x /etc/periodic/15min/feed-cron + +WORKDIR /app + +CMD ["/init.sh"] + diff --git a/README.md b/README.md index b8376f6..5c0c7b4 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,14 @@ -# RSS to ActivityPub Converter +# rss-to-activitypub +An RSS to ActivityPub converter. -This is a server that lets users convert any RSS feed to an ActivityPub actor that can be followed by users on ActivityPub-compliant social networks like Mastodon. -This is based on my [Express ActivityPub Server](https://github.com/dariusk/express-activitypub), a simple Node/Express server that supports a subset of ActivityPub. - -## Requirements - -This requires Node.js v10.10.0 or above. - -You also need `beanstalkd` running. This is a simple and fast queueing system we use to manage polling RSS feeds. [Here are installation instructions](https://beanstalkd.github.io/download.html). On a production server you'll want to [install it as a background process](https://github.com/beanstalkd/beanstalkd/tree/master/adm). - -## Installation - -Clone the repository, then `cd` into its root directory. Install dependencies: - -`npm i` - -Then copy `config.json.template` to `config.json`: - -`cp config.json.template config.json` - -Update your new `config.json` file: - -```js -{ - "DOMAIN": "mydomain.com", - "PORT_HTTP": "3000", - "PORT_HTTPS": "8443", - "PRIVKEY_PATH": "/path/to/your/ssl/privkey.pem", - "CERT_PATH": "/path/to/your/ssl/cert.pem" -} +To build: +just run: ``` - -* `DOMAIN`: your domain! this should be a discoverable domain of some kind like "example.com" or "rss.example.com" -* `PORT_HTTP`: the http port that Express runs on -* `PORT_HTTPS`: the https port that Express runs on -* `PRIVKEY_PATH`: point this to your private key you got from Certbot or similar -* `CERT_PATH`: point this to your cert you got from Certbot or similar - -Run the server! - -`node index.js` - -Go to `https://whateveryourdomainis.com:3000/convert` or whatever port you selected for HTTP, and enter an RSS feed and a username. If all goes well it will create a new ActivityPub user with instructions on how to view the user. - -## Sending out updates to followers - -There is also a file called `queueFeeds.js` that needs to be run on a cron job or similar scheduler. I like to run mine once a minute. It queries every RSS feed in the database to see if there has been a change to the feed. If there is a new post, it sends out the new post to everyone subscribed to its corresponding ActivityPub Actor. - -## Local testing - -You can use a service like [ngrok](https://ngrok.com/) to test things out before you deploy on a real server. All you need to do is install ngrok and run `ngrok http 3000` (or whatever port you're using if you changed it). Then go to your `config.json` and update the `DOMAIN` field to whatever `abcdef.ngrok.io` domain that ngrok gives you and restart your server. - -Then make sure to manually run `updateFeed.js` when the feed changes. I recommend having your own test RSS feed that you can update whenever you want. - -## Database - -This server uses a SQLite database stored in the file `bot-node.db` to keep track of all the data. To connect directly to the database for debugging, from the root directory of the project, run: - -```bash -sqlite3 bot-node.db +docker build -t rss-to-activity-pub . ``` +and then +docker-compose up -There are two tables in the database: `accounts` and `feeds`. - -### `accounts` - -This table keeps track of all the data needed for the accounts. Columns: - -* `name` `TEXT PRIMARY KEY`: the account name, in the form `thename@example.com` -* `privkey` `TEXT`: the RSA private key for the account -* `pubkey` `TEXT`: the RSA public key for the account -* `webfinger` `TEXT`: the entire contents of the webfinger JSON served for this account -* `actor` `TEXT`: the entire contents of the actor JSON served for this account -* `apikey` `TEXT`: the API key associated with this account -* `followers` `TEXT`: a JSON-formatted array of the URL for the Actor JSON of all followers, in the form `["https://remote.server/users/somePerson", "https://another.remote.server/ourUsers/anotherPerson"]` -* `messages` `TEXT`: not yet used but will eventually store all messages so we can render them on a "profile" page - -### `feeds` - -This table keeps track of all the data needed for the feeds. Columns: - -* `feed` `TEXT PRIMARY KEY`: the URI of the RSS feed -* `username` `TEXT`: the username associated with the RSS feed -* `content` `TEXT`: the most recent copy fetched of the RSS feed's contents - -## License - -Copyright (c) 2018 Darius Kazemi. Licensed under the MIT license. +docker-compose.yml is example for version with Treafik reverse proxy +also edit app/views/home.pug couse is our rebranded version for rss.fediwersum.pl diff --git a/.gitignore b/app/.gitignore similarity index 100% rename from .gitignore rename to app/.gitignore diff --git a/LICENSE-MIT b/app/LICENSE-MIT similarity index 100% rename from LICENSE-MIT rename to app/LICENSE-MIT diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..b8376f6 --- /dev/null +++ b/app/README.md @@ -0,0 +1,90 @@ +# RSS to ActivityPub Converter + +This is a server that lets users convert any RSS feed to an ActivityPub actor that can be followed by users on ActivityPub-compliant social networks like Mastodon. + +This is based on my [Express ActivityPub Server](https://github.com/dariusk/express-activitypub), a simple Node/Express server that supports a subset of ActivityPub. + +## Requirements + +This requires Node.js v10.10.0 or above. + +You also need `beanstalkd` running. This is a simple and fast queueing system we use to manage polling RSS feeds. [Here are installation instructions](https://beanstalkd.github.io/download.html). On a production server you'll want to [install it as a background process](https://github.com/beanstalkd/beanstalkd/tree/master/adm). + +## Installation + +Clone the repository, then `cd` into its root directory. Install dependencies: + +`npm i` + +Then copy `config.json.template` to `config.json`: + +`cp config.json.template config.json` + +Update your new `config.json` file: + +```js +{ + "DOMAIN": "mydomain.com", + "PORT_HTTP": "3000", + "PORT_HTTPS": "8443", + "PRIVKEY_PATH": "/path/to/your/ssl/privkey.pem", + "CERT_PATH": "/path/to/your/ssl/cert.pem" +} +``` + +* `DOMAIN`: your domain! this should be a discoverable domain of some kind like "example.com" or "rss.example.com" +* `PORT_HTTP`: the http port that Express runs on +* `PORT_HTTPS`: the https port that Express runs on +* `PRIVKEY_PATH`: point this to your private key you got from Certbot or similar +* `CERT_PATH`: point this to your cert you got from Certbot or similar + +Run the server! + +`node index.js` + +Go to `https://whateveryourdomainis.com:3000/convert` or whatever port you selected for HTTP, and enter an RSS feed and a username. If all goes well it will create a new ActivityPub user with instructions on how to view the user. + +## Sending out updates to followers + +There is also a file called `queueFeeds.js` that needs to be run on a cron job or similar scheduler. I like to run mine once a minute. It queries every RSS feed in the database to see if there has been a change to the feed. If there is a new post, it sends out the new post to everyone subscribed to its corresponding ActivityPub Actor. + +## Local testing + +You can use a service like [ngrok](https://ngrok.com/) to test things out before you deploy on a real server. All you need to do is install ngrok and run `ngrok http 3000` (or whatever port you're using if you changed it). Then go to your `config.json` and update the `DOMAIN` field to whatever `abcdef.ngrok.io` domain that ngrok gives you and restart your server. + +Then make sure to manually run `updateFeed.js` when the feed changes. I recommend having your own test RSS feed that you can update whenever you want. + +## Database + +This server uses a SQLite database stored in the file `bot-node.db` to keep track of all the data. To connect directly to the database for debugging, from the root directory of the project, run: + +```bash +sqlite3 bot-node.db +``` + +There are two tables in the database: `accounts` and `feeds`. + +### `accounts` + +This table keeps track of all the data needed for the accounts. Columns: + +* `name` `TEXT PRIMARY KEY`: the account name, in the form `thename@example.com` +* `privkey` `TEXT`: the RSA private key for the account +* `pubkey` `TEXT`: the RSA public key for the account +* `webfinger` `TEXT`: the entire contents of the webfinger JSON served for this account +* `actor` `TEXT`: the entire contents of the actor JSON served for this account +* `apikey` `TEXT`: the API key associated with this account +* `followers` `TEXT`: a JSON-formatted array of the URL for the Actor JSON of all followers, in the form `["https://remote.server/users/somePerson", "https://another.remote.server/ourUsers/anotherPerson"]` +* `messages` `TEXT`: not yet used but will eventually store all messages so we can render them on a "profile" page + +### `feeds` + +This table keeps track of all the data needed for the feeds. Columns: + +* `feed` `TEXT PRIMARY KEY`: the URI of the RSS feed +* `username` `TEXT`: the username associated with the RSS feed +* `content` `TEXT`: the most recent copy fetched of the RSS feed's contents + +## License + +Copyright (c) 2018 Darius Kazemi. Licensed under the MIT license. diff --git a/config.json.template b/app/config.json.template similarity index 100% rename from config.json.template rename to app/config.json.template diff --git a/app/data/.gitignore b/app/data/.gitignore new file mode 100644 index 0000000..f935021 --- /dev/null +++ b/app/data/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/index.js b/app/index.js similarity index 97% rename from index.js rename to app/index.js index ed23097..288d360 100644 --- a/index.js +++ b/app/index.js @@ -3,7 +3,7 @@ const { DOMAIN, PRIVKEY_PATH, CERT_PATH, PORT_HTTP, PORT_HTTPS } = config; const express = require('express'); const app = express(); const Database = require('better-sqlite3'); -const db = new Database('bot-node.db'); +const db = new Database('data/bot-node.db'); const fs = require('fs'); const routes = require('./routes'), bodyParser = require('body-parser'), diff --git a/package.json b/app/package.json similarity index 100% rename from package.json rename to app/package.json diff --git a/public/convert/index.html b/app/public/convert/index.html similarity index 100% rename from public/convert/index.html rename to app/public/convert/index.html diff --git a/queueFeeds.js b/app/queueFeeds.js similarity index 77% rename from queueFeeds.js rename to app/queueFeeds.js index 2187766..4712dd9 100644 --- a/queueFeeds.js +++ b/app/queueFeeds.js @@ -1,9 +1,10 @@ const Database = require('better-sqlite3'); -const db = new Database('bot-node.db'); +const db = new Database('data/bot-node.db'); const Jackd = require('jackd'); const beanstalkd = new Jackd(); + async function foo() { // get all feeds from DB @@ -13,7 +14,7 @@ async function foo() { let count = 0; - await beanstalkd.connect() + await beanstalkd.connect({host: 'beanstalkd', port: 11300 }) for (feed of feeds) { await beanstalkd.put(feed.feed) diff --git a/routes/api.js b/app/routes/api.js similarity index 100% rename from routes/api.js rename to app/routes/api.js diff --git a/routes/inbox.js b/app/routes/inbox.js similarity index 100% rename from routes/inbox.js rename to app/routes/inbox.js diff --git a/routes/index.js b/app/routes/index.js similarity index 100% rename from routes/index.js rename to app/routes/index.js diff --git a/routes/message.js b/app/routes/message.js similarity index 100% rename from routes/message.js rename to app/routes/message.js diff --git a/routes/user.js b/app/routes/user.js similarity index 100% rename from routes/user.js rename to app/routes/user.js diff --git a/routes/webfinger.js b/app/routes/webfinger.js similarity index 100% rename from routes/webfinger.js rename to app/routes/webfinger.js diff --git a/updateFeeds.js b/app/updateFeeds.js similarity index 99% rename from updateFeeds.js rename to app/updateFeeds.js index c9c4f2c..288ae9a 100644 --- a/updateFeeds.js +++ b/app/updateFeeds.js @@ -1,7 +1,7 @@ const config = require('./config.json'); const { DOMAIN, PRIVKEY_PATH, CERT_PATH, PORT_HTTP, PORT_HTTPS } = config; const Database = require('better-sqlite3'); -const db = new Database('bot-node.db'), +const db = new Database('data/bot-node.db'), Parser = require('rss-parser'), request = require('request'), crypto = require('crypto'), @@ -10,7 +10,7 @@ const db = new Database('bot-node.db'), const Jackd = require('jackd'); const beanstalkd = new Jackd(); -beanstalkd.connect() +beanstalkd.connect({host: 'beanstalkd',port: 11300}) async function foo() { while (true) { diff --git a/app/views/home.pug b/app/views/home.pug new file mode 100644 index 0000000..16605ab --- /dev/null +++ b/app/views/home.pug @@ -0,0 +1,15 @@ +html + head + + title RSS2Fediwersum + style + include style.css + body + h1 Kanały RSS do Fediwersum (rss2fedi) + f4 od zespołu Lhub.pl + p To jest serwis oferujący Boty ActivityPub konwertujące kanały RSS do strumienia powiadomień. Możesz je subskrybować z Mastodona lub innych aplikacji zgodnych z Fediwersum/ActivityPub. + p Lista botów Rss2Fedi + p Chcesz nam zgłosić kolejny feed RSS do dodania? Napisz lub kliknij do @m0bi@lhub.social + p Bazuje na projekcie RSS to ActivityPub Converter (Kod) od Darius Kazemi. + + diff --git a/views/message.pug b/app/views/message.pug similarity index 100% rename from views/message.pug rename to app/views/message.pug diff --git a/views/style.css b/app/views/style.css similarity index 100% rename from views/style.css rename to app/views/style.css diff --git a/views/user.pug b/app/views/user.pug similarity index 100% rename from views/user.pug rename to app/views/user.pug diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7814a4d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,62 @@ +version: '3' +services: + app: + restart: unless-stopped + image: rss-to-activity-pub + environment: + - DOMAIN=rss.fediwersum.pl + labels: + traefik.enable: "true" +# traefik.http.routers.rss-to-activity-pub-conv.entrypoints: websecure +# traefik.http.routers.rss-to-activity-pub-conv.rule: (Host(`rss.fediwersum.pl`) && PathPrefix(`/convert`)) +# traefik.http.routers.rss-to-activity-pub-conv.middlewares: rss-to-activity-pub-auth +# traefik.http.routers.rss-to-activity-pub-conv.tls.certresolver: le +# traefik.http.middlewares.rss-to-activity-pub-auth.basicauth.users: some:user + traefik.http.routers.rss-to-activity-pub.entrypoints: websecure + traefik.http.routers.rss-to-activity-pub.rule: Host(`rss.fediwersum.pl`) + traefik.http.routers.rss-to-activity-pub.tls.certresolver: le + traefik.http.services.rss-to-activity-pub.loadbalancer.server.port: 3000 + networks: + - frontend + - beanstalk + volumes: + - data:/app/data + + cron: + restart: unless-stopped + image: rss-to-activity-pub + environment: + - DOMAIN=rss.fediwersum.pl + command: /usr/sbin/crond -f -l 0 + volumes: + - data:/app/data + networks: + - beanstalk + + update-feeds: + restart: unless-stopped + image: rss-to-activity-pub + environment: + - DOMAIN=rss.fediwersum.pl + command: /init-update.sh + volumes: + - data:/app/data + networks: + - beanstalk + + beanstalkd: + restart: unless-stopped + image: schickling/beanstalkd + networks: + - beanstalk + + +networks: + frontend: + external: true + beanstalk: + driver: overlay + +volumes: + data: + driver: local diff --git a/feed-cron b/feed-cron new file mode 100755 index 0000000..afb24de --- /dev/null +++ b/feed-cron @@ -0,0 +1 @@ +/bin/sh -c "cd /app && node queueFeeds.js" >/dev/null 2>&1 diff --git a/init-update.sh b/init-update.sh new file mode 100755 index 0000000..63127bb --- /dev/null +++ b/init-update.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +cd /app + +cp config.json.template config.json + +if [[ ! -z ${DOMAIN+x} ]]; then + sed -i "s/\"DOMAIN\": \"\"/\"DOMAIN\": \"${DOMAIN}\"/g" config.json +else + echo 'Env var $DOMAIN is not set. Set it before starting the container' + exit 1 +fi + +if [[ ! -z ${PORT_HTTP+x} ]]; then + sed -i "s/\"PORT_HTTP\": \"3000\"/\"PORT_HTTP\": \"${PORT_HTTP}\"/g" config.json +fi + +if [[ ! -z ${PORT_HTTPS+x} ]]; then + sed -i "s/\"PORT_HTTPS\": \"8443\"/\"PORT_HTTPS\": \"${PORT_HTTPS}\"/g" config.json +fi + +if [[ ! -z ${PRIVKEY_PATH+x} ]]; then + sed -i "s@\"PRIVKEY_PATH\": \"\"@\"PRIVKEY_PATH\": \"${PRIVKEY_PATH}\"@g" config.json +fi + +if [[ ! -z ${CERT_PATH+x} ]]; then + sed -i "s@\"CERT_PATH\": \"\"@\"CERT_PATH\": \"${CERT_PATH}\"@g" config.json +fi + +echo "Configuration file :" + +cat config.json + + +node updateFeeds.js + diff --git a/init.sh b/init.sh new file mode 100755 index 0000000..9a61842 --- /dev/null +++ b/init.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +cd /app + +cp config.json.template config.json + +if [[ ! -z ${DOMAIN+x} ]]; then + sed -i "s/\"DOMAIN\": \"\"/\"DOMAIN\": \"${DOMAIN}\"/g" config.json +else + echo 'Env var $DOMAIN is not set. Set it before starting the container' + exit 1 +fi + +if [[ ! -z ${PORT_HTTP+x} ]]; then + sed -i "s/\"PORT_HTTP\": \"3000\"/\"PORT_HTTP\": \"${PORT_HTTP}\"/g" config.json +fi + +if [[ ! -z ${PORT_HTTPS+x} ]]; then + sed -i "s/\"PORT_HTTPS\": \"8443\"/\"PORT_HTTPS\": \"${PORT_HTTPS}\"/g" config.json +fi + +if [[ ! -z ${PRIVKEY_PATH+x} ]]; then + sed -i "s@\"PRIVKEY_PATH\": \"\"@\"PRIVKEY_PATH\": \"${PRIVKEY_PATH}\"@g" config.json +fi + +if [[ ! -z ${CERT_PATH+x} ]]; then + sed -i "s@\"CERT_PATH\": \"\"@\"CERT_PATH\": \"${CERT_PATH}\"@g" config.json +fi + +echo "Configuration file :" + +cat config.json + + +node index.js + diff --git a/views/home.pug b/views/home.pug deleted file mode 100644 index 6a5457b..0000000 --- a/views/home.pug +++ /dev/null @@ -1,11 +0,0 @@ -html - head - - title Feed Converter - style - include style.css - body - h1 RSS to ActivityPub Converter - p.account by Darius Kazemi - p This is a service to convert any RSS feed into an account that Mastodon (or any other ActivityPub social network) can subscribe to. Source code here. - p Click here to start!