glit.sh setup notes

This document describes how you could set up a server like glit.sh yourself. It’s also a reminder of how I set this server up, so that I can figure out what’s going on when I look back at this in a week ;)

It’s still a work in progress - see my TODO list for the stuff I want to get to but haven’t yet :)

I’m running on a DigitalOcean $5/month machine, running Debian GNU/Linux 10 in the nyc3 region. I have DigitalOcean’s weekly backups turned on in addition to the daily tarsnap backups described below for redundancy. DO NOT use a floating IP - you will not be able to send outbound mail if you do. The name of the droplet should be the hostname that you will be sending mail from, so that the PTR record is set correctly.

Note that there may be some path-dependency issues in this document, particularly around the HTTPS setup - I’m not describing things in exactly the order I did them, since a lot of this doc pulls info and config files from the running system.

dns setup

web server setup

install software

apt update
apt upgrade
apt install lighttpd git socat # important
apt install curl telnet man htop tmux strace lsof expect make dtrx pandoc # just for fun (and rendering this doc!)

HTTPS Setup

Do this as root, in your home directory (/root/).

git clone https://github.com/Neilpang/acme.sh.git
(cd ./acme.sh && ./acme.sh --install)
mkdir -vp /var/www/html/.well-known/acme-challenge/
chown -R www-data:www-data /var/www/html/.well-known/acme-challenge/
chmod -R 0555 /var/www/html/.well-known/acme-challenge/
mkdir -p /etc/lighttpd/ssl/glit.sh/
(cd /etc/lighttpd/ssl/glit.sh/ && openssl dhparam -out dhparam.pem -dsaparam 4096)
# log out and in again to get acme.sh in your path
acme.sh --issue -w /var/www/html -d glit.sh -d src.glit.sh -k 4096

Set /root/.acme.sh/glit.sh/hook.sh to contain:

#!/bin/bash

DOMAIN="glit.sh"
DEST="/etc/lighttpd/ssl/${DOMAIN}"
CROOT="/root/.acme.sh/${DOMAIN}"
 
sslfile="${DEST}/ssl.pem"
certfile="${CROOT}/${DOMAIN}.cer"
keyfile="${CROOT}/${DOMAIN}.key"
 
cat "${certfile}" "${keyfile}" > "${sslfile}"
systemctl restart lighttpd

Then run:

chmod +x .acme.sh/glit.sh/hook.sh
acme.sh --installcert -d glit.sh -d src.glit.sh \
        --capath /etc/lighttpd/ssl/glit.sh/ca.cer \
        --reloadcmd /root/.acme.sh/glit.sh/hook.sh

lighttpd setup

rm /etc/lighttpd/conf-enabled/99-unconfigured.conf
touch /var/log/lighttpd/access.log
chown www-data:www-data /var/log/lighttpd/access.log

Set /etc/lighttpd/lighttpd.conf to contain the following:

server.modules = (
    "mod_expire",
    "mod_indexfile",
    "mod_cgi",
    "mod_access",
    "mod_accesslog",
    "mod_alias",
    "mod_redirect",
    "mod_setenv",
    "mod_userdir",
    "mod_openssl",
    "mod_rewrite",
    "mod_compress",
    "mod_dirlisting",
    "mod_staticfile",
)

server.document-root = "/var/www/html/"
server.upload-dirs = ( "/var/cache/lighttpd/uploads" )
server.errorlog = "/var/log/lighttpd/error.log"
server.breakagelog = "/var/log/lighttpd/breakage.log"
server.pid-file = "/var/run/lighttpd.pid"
server.username = "www-data"
server.groupname = "www-data"
server.bind = "0.0.0.0"
server.port = 80

accesslog.filename = "/var/log/lighttpd/access.log" 

userdir.path = "public_html"

server.dir-listing = "enable"
dir-listing.encoding = "utf-8"
dir-listing.external-css = "/dir.css"

server.http-parseopts = (
    "header-strict" => "enable",
    "host-strict" => "enable",
    "host-normalize" => "enable",
    "url-normalize-unreserved" => "enable",
    "url-normalize-required" => "enable",
    "url-ctrls-reject" => "enable",
    "url-path-2f-decode" => "enable",
    "url-path-dotseg-remove" => "enable",
)

index-file.names = (
    "index.sh",
    "main.sh",
    "index.py",
    "main.py",
    "index.pl",
    "main.pl",
    "index.html",
    "main.html",
)

url.access-deny = ( "~", ".inc" )
static-file.exclude-extensions = ( ".pl", ".py", ".sh" )

compress.cache-dir = "/var/cache/lighttpd/compress/"
compress.filetype = ( "application/javascript", "text/css", "text/html", "text/plain" )

include_shell "/usr/share/lighttpd/create-mime.conf.pl"

$SERVER["socket"] == "[::]:80" {
    $HTTP["host"] =~ ".*" {
        url.redirect = (".*" => "https://%0$0")
    }
}

$SERVER["socket"] == "0.0.0.0:80" {
    $HTTP["host"] =~ ".*" {
        url.redirect = (".*" => "https://%0$0")
    }
}

$SERVER["socket"] == "0.0.0.0:443" {
    ssl.engine = "enable"
    ssl.disable-client-renegotiation = "enable"

    ssl.pemfile = "/etc/lighttpd/ssl/glit.sh/ssl.pem"
    ssl.ca-file = "/etc/lighttpd/ssl/glit.sh/ca.cer"
    ssl.dh-file = "/etc/lighttpd/ssl/glit.sh/dhparam.pem"

    ssl.ec-curve = "secp384r1"

    setenv.add-environment = (
        "HTTPS" => "on"
    )

    ssl.use-sslv2 = "disable"
    ssl.use-sslv3 = "disable"
    ssl.openssl.ssl-conf-cmd = ("Protocol" => "-TLSv1.1, -TLSv1, -SSLv3")

    ssl.cipher-list = "EECDH+AESGCM:EDH+AESGCM"

    setenv.add-response-header = (
        "Strict-Transport-Security" => "max-age=31536000; includeSubDomains; preload"
    )
}

$SERVER["socket"] == "[::]:443" {
    ssl.engine = "enable"
    ssl.disable-client-renegotiation = "enable"

    ssl.pemfile = "/etc/lighttpd/ssl/glit.sh/ssl.pem"
    ssl.ca-file = "/etc/lighttpd/ssl/glit.sh/ca.cer"
    ssl.dh-file = "/etc/lighttpd/ssl/glit.sh/dhparam.pem"

    ssl.ec-curve = "secp384r1"

    setenv.add-environment = (
        "HTTPS" => "on"
    )

    ssl.use-sslv2 = "disable"
    ssl.use-sslv3 = "disable"
    ssl.openssl.ssl-conf-cmd = ("Protocol" => "-TLSv1.1, -TLSv1, -SSLv3")

    ssl.cipher-list = "EECDH+AESGCM:EDH+AESGCM"

    setenv.add-response-header = (
        "Strict-Transport-Security" => "max-age=31536000; includeSubDomains; preload"
    )
}

# TODO - does this stop https cert renewal from working?
# yes. it does. also, the http redirect above as well :/
alias.url = ( "/" => "/home/wesleyac/public_html/homepage/" )

$HTTP["host"] == "glit.sh" {
    cgi.assign = (
        ".py"  => "",
        ".sh" => "",
        ".pl" => "",
        ".cgi" => "",
    )
}

$HTTP["host"] == "src.glit.sh" {
    setenv.add-response-header = (
        "Access-Control-Allow-Origin" => "*",
        "Access-Control-Allow-Methods" => "HEAD, GET, OPTIONS",
        "Access-Control-Expose-Headers" => "Content-Range, Date, Etag, Cache-Control, Last-Modified",
        "Access-Control-Allow-Headers" => "Content-Type, Origin, Accept, Range, Cache-Control",
        "Timing-Allow-Origin" => "*",
        "Content-Type" => "text/plain; charset=UTF-8",
        "Cache-Control" => "no-store",
    )
    expire.url = ( "" => "access plus 0 seconds" )
    cgi.assign = (
        ".py"  => "/root/viewsrc.sh",
        ".sh" => "/root/viewsrc.sh",
        ".pl" => "/root/viewsrc.sh",
        ".cgi" => "/root/viewsrc.sh",
    )
}

This gets me an A+ on the SSL Labs test :)

view source details

make a /root/viewsrc.sh file:

#!/bin/sh
echo ""
cat $1

this is to prevent shebangs with colons in them from being treated as HTTP headers

postfix setup

apt install mailutils postfix # select "Internet Site" in postfix config

edit /etc/postfix/main.cf to contain the following (change the mydestination line):

smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
biff = no

# appending .domain is the MUA's job.
append_dot_mydomain = no

readme_directory = no

compatibility_level = 2

# TLS parameters
smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
smtpd_use_tls=yes
smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache

smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
myhostname = glit.sh
alias_maps = hash:/etc/aliases
alias_database = hash:/etc/aliases
myorigin = /etc/mailname
mydestination = $myhostname, localhost.$mydomain, $mydomain
relayhost = 
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
mailbox_size_limit = 0
recipient_delimiter = +
inet_interfaces = all
inet_protocols = all

# from inside the chroot, the socket will be in /var/run/opendkim 
smtpd_milters = unix:/var/run/opendkim/opendkim.sock
non_smtpd_milters = unix:/var/run/opendkim/opendkim.sock
milter_default_action = accept
milter_protocol = 6

restart postfix:

systemctl restart postfix

try to send mail:

echo "<3 <3 <3" | mail -s "Glit.sh test email" me@wesleyac.com

discover that you’re on the CBL blacklist. Remove host from CBL blacklist. Try to send mail again. Discover that you’re greylisted by Fastmail. Cry. The SparkPost Authentication Checker may be useful for verifying that SPF is set up correctly.

dkim setup

apt install opendkim opendkim-tools
opendkim-genkey -D /etc/dkimkeys/ -d glit.sh -s mail
chown opendkim:opendkim /etc/dkimkeys/*
chmod go-rwx /etc/dkimkeys/*
mkdir -p /var/spool/postfix/var/run/opendkim
chown opendkim. /var/spool/postfix/var/run/opendkim
chmod go-rwx /var/spool/postfix/var/run/opendkim
chmod g+x /var/spool/postfix/var/run/opendkim

edit /etc/opendkim.conf:

# Log to syslog
Syslog yes
# Required to use local socket with MTAs that access the socket as a non-
# privileged user (e.g. Postfix)
UMask 007
UserID opendkim
PidFile /var/run/opendkim/opendkim.pid

Canonicalization simple
Mode sv
SubDomains no

# Postfix runs inside /var/spool/postfix/ chroot
Socket local:/var/spool/postfix/var/run/opendkim/opendkim.sock

KeyTable file:/etc/dkimkeys/keytable
SigningTable refile:/etc/dkimkeys/signingtable 
InternalHosts refile:/etc/dkimkeys/trustedhosts

# Always oversign From (sign using actual From and a null From to prevent
# malicious signatures header fields (From and/or others) between the signer
# and the verifier.  From is oversigned by default in the Debian pacakge
# because it is often the identity key used by reputation systems and thus
# somewhat security sensitive.
OversignHeaders From

# Specifies a file from which trust anchor data should be read when doing
# DNS queries and applying the DNSSEC protocol.  See the Unbound documentation
# at http://unbound.net for the expected format of this file.
TrustAnchorFile /usr/share/dns/root.key

create /etc/dkimkeys/keytable:

mail._domainkey.glit.sh glit.sh:mail:/etc/dkimkeys/mail.private 

create /etc/dkimkeys/signingtable:

*@glit.sh mail._domainkey.glit.sh

create /etc/dkimkeys/trustedhosts:

127.0.0.1
systemctl restart opendkim
adduser postfix opendkim

Make a DNS TXT record based on /etc/dkimkeys/mail.txt.

firewall setup

apt install ufw
ufw allow ssh
ufw allow http
ufw allow https
ufw allow 25
ufw enable

add a user!

adduser wesleyac
usermod -aG sudo wesleyac
mkdir /home/wesleyac/.ssh
chown wesleyac:wesleyac /home/wesleyac/.ssh/
cp .ssh/authorized_keys /home/wesleyac/.ssh/authorized_keys
chown wesleyac:wesleyac /home/wesleyac/.ssh/authorized_keys

there’s a script at /root/mkusers.py that listens on localhost:420 for requests to add users:

#!/usr/bin/env python3

import crypt, json, os, stat, shutil, subprocess, base64, struct, binascii, pwd
from http.server import BaseHTTPRequestHandler, HTTPServer

def check_ssh_key(key):
    key = key.split()

    if len(key) != 3:
        return (False, "must have key type, key data, and username")

    try:
        data = base64.decodestring(bytes(key[1], 'utf-8'))
        str_len = struct.unpack('>I', data[:4])[0]
        assert(int(str_len) == 7)
    except:
        return (False, "could not decode data")

    if data[4:11] != bytes(key[0], 'utf-8'):
        return (False, "key type doesn't match data")

    return (True, "")

class Server(BaseHTTPRequestHandler):
    def do_POST(self):
        content_length = int(self.headers['Content-Length'])
        data = json.loads(self.rfile.read(content_length).decode())
        print(data)
        errs = []
        try:
            user = data["username"]
            email = data["email"]
            passwd = data["password"]
            ssh_key = data["ssh"]
        except:
            errs.append("couldn't find key")
        valid_key = check_ssh_key(ssh_key)
        if not valid_key[0]:
            errs.append(f"ssh key error: {valid_key[1]}")
        if user in [str(x.pw_name) for x in pwd.getpwall()]:
            errs.append("username already taken")
        if len(user) > 30:
            errs.append("username too long")
        if not (user.isidentifier() and user.islower()):
            errs.append("disallowed characters in username :(")
        if len(errs) != 0:
            response = json.dumps({"result": "err", "errs": errs}).encode()
        else:
            subprocess.call(["useradd", "-p", crypt.crypt(passwd), "-c", email, "-m", user])

            os.makedirs(f"/home/{user}/.ssh/", exist_ok=True)
            with open(f"/home/{user}/.ssh/authorized_keys", 'w') as authorized_keys:
                authorized_keys.write(ssh_key)
            shutil.chown(f"/home/{user}/.ssh/", user, user)
            shutil.chown(f"/home/{user}/.ssh/authorized_keys", user, user)

            os.makedirs(f"/home/{user}/public_html/", exist_ok=True)
            with open(f"/home/{user}/public_html/index.sh", 'w') as index:
                index.write(f"""#!/bin/sh
cat <<EOF
<!DOCTYPE html>
<html>
\t<head>
\t\t<title>{user}'s page!</title>
\t</head>
\t<body>
\t\t<h1>this is {user}'s page!</h1>

\t\t<p>this is the default index page - try changing it in the <a href="https://glit.sh/~wesleyac/editor/#~{user}/index.sh">web editor</a>!</p>

\t\t<p>the current time is $(date)</p>
\t</body>
</html>
EOF
""")
            shutil.chown(f"/home/{user}/public_html/", user, user)
            shutil.chown(f"/home/{user}/public_html/index.sh", user, user)
            os.chmod(f"/home/{user}/public_html/index.sh", 0o775)
        
            response = json.dumps({"result": "ok", "errs": []}).encode()

        self.send_response(200)
        self.send_header('Content-Type', 'application/json')
        self.send_header('Content-Length', len(response))
        self.end_headers()
        self.wfile.write(response)
        self.wfile.flush()

if __name__ == "__main__":
    s = HTTPServer(("", 420), Server)
    s.serve_forever()

and a systemd file to start it on boot at /lib/systemd/system/mkusers.service:

[Unit]
Description=Make user accounts

[Service]
Type=simple
ExecStart=/root/mkusers.py
User=root

[Install]
WantedBy=multi-user.target

which you can enable via:

systemctl start mkusers
systemctl enable mkusers

disable ssh root and password login

(i might undo this soon, this is just while i have really weak passwords set for testing)

edit /etc/ssh/sshd_config:

ChallengeResponseAuthentication no
PasswordAuthentication no
UsePAM no
PermitRootLogin no
sudo systemctl reload ssh

set up backups via tarsnap

apt install gpg
wget https://pkg.tarsnap.com/tarsnap-deb-packaging-key.asc
# verify signature
apt-key add tarsnap-deb-packaging-key.asc
echo "deb http://pkg.tarsnap.com/deb/$(lsb_release -s -c) ./" | sudo tee -a /etc/apt/sources.list.d/tarsnap.list
apt-get update
apt install tarsnap
tarsnap-keygen --keyfile /root/tarsnap-main.key --user me@wesleyac.com --machine glitsh
tarsnap-keymgmt --outkeyfile tarsnap.key -w tarsnap-main.key
# save tarsnap-main.key somewhere safe then delete it from the server

make a /root/tarsnap-backup.sh file:

#!/bin/sh
tarsnap -c -f "$(uname -n)-$(date +%Y-%m-%d_%H-%M-%S)" /home/ /root/ /etc/ /lib/systemd/system/ /var/mail/ /var/spool/ /var/log/ /var/www/

and go ahead and run it to check that it works (including checking that backup restore works!)

and to /etc/crontab, add:

11 4 * * * root /root/tarsnap-backup.sh

to make a daily backup task. check back later to see that it worked.

you’re done!

enjoy <3