Devuan / Debian cache with varnish

How to add a high performance devuan / debian / ubuntu / cpan repository cache (merged) with the security repo.
Clean access paths:

• /devuan (merged devuan)
• /debian (debian)
• /debian-security (debian security)
• /ubuntu (ubuntu)
• /ubuntu-security (ubuntu security)
• /cpan (bonus)

If none of these paths matches, varnish falls back to the local backend on port 8080. You can add a http server running on localhost:8080 and host your own apt repo like this.

Let’s say your varnish will be accessible via the hostname ‘foo.bar.net’.
You have to replace all occurences of ‘foo.bar.net’ from the configuration with your hostname.

# apt-get install varnish

Edit /etc/varnish/default.vcl:

vcl 4.0;

backend default {
        .host = "127.0.0.1";
        .port = "8080";
}

backend debian {
        .host = "ftp.debian.org";
        .port = "80";
}

backend ubuntu {
        # RR-DNS on us.archive.ubuntu.com... choose one
        .host = "91.189.91.26";
        .port = "80";
}

backend ubuntusec {
        # RR-DNS on security.ubuntu.com... choose one
        .host = "91.189.88.149";
        .port = "80";
}

backend cpan {
        .host = "www.cpan.org";
        .port = "80";
}

backend debiansec {
        # RR-DNS on security.debian.org... choose one
        .host = "217.196.149.233";
        .port = "80";
}

backend devuan {
        .host = "auto.mirror.devuan.org";
        .port = "80";
}


sub vcl_recv {
        if (req.url ~ "^/cpan") {
		set req.backend_hint = cpan;
		set req.url =  regsub( req.url, "^/cpan/", "/");
                set req.http.host = "www.cpan.org";

        } elsif (req.url ~ "^/devuan") {
                if (req.url ~ "^/devuan/pool/DEBIAN-SECURITY") {
                        set req.url = regsub(
                                        req.url,
                                        "^/devuan/pool/DEBIAN-SECURITY/",
                                        "http://foo.bar.net/security/pool/");
                        return ( synth (
                                        750,
                                        req.url
                                       ));
                }
                elsif (req.url ~ "^/devuan/pool/DEBIAN") {
                        set req.url = regsub(
                                        req.url,
                                        "^/devuan/pool/DEBIAN/",
                                        "http://foo.bar.net/debian/pool/");
                        return ( synth (
                                        750,
                                        req.url
                                       ));
                }
                else {
                        set req.url =  regsub(
                                        req.url,
                                        "^/devuan/",
                                        "/merged/");
                }
                set req.backend_hint = devuan;
                set req.http.host = "auto.mirror.devuan.org";

        } elsif (req.url ~ "^/debian/") {
                set req.backend_hint = debian;
                set req.http.host = "ftp.debian.org";

        } elsif (req.url ~ "^/ubuntu/") {
                set req.backend_hint = ubuntu;
                set req.http.host = "us.archive.ubuntu.com";

        } elsif (req.url ~ "^/ubuntu-security/") {
                set req.backend_hint = ubuntusec;
                set req.http.host = "security.ubuntu.com";
                set req.url = regsub( req.url, "^/ubuntu-security/", "/ubuntu/" );

        } elsif (req.url ~ "^/debian-security/") {
                set req.backend_hint = debiansec;
                set req.http.host = "security.debian.org";
                set req.url = regsub( req.url, "-updates/", "/updates/" );

        } elsif (req.url ~ "^/security/") {
                set req.backend_hint = debiansec;
                set req.http.host = "security.debian.org";
                set req.url = regsub( req.url, "^/security/", "/debian-security/" );
                set req.url = regsub( req.url, "-updates/", "/updates/" );
        }
}

sub vcl_backend_response {
        # Don't cache dists
        if (bereq.url ~ "^/(devuan|debian[^/]*|ubuntu[^/]*|merged)/dists") {
                set beresp.ttl = 120s;
                set beresp.uncacheable = true;
                return (deliver);

        } elsif (bereq.url ~ "/modules/.*.gz") {
		# CPAN
		set beresp.ttl = 15s;
		set beresp.uncacheable = true;
		return (deliver);
	}
}

sub vcl_deliver {
}

sub vcl_synth {
        if (resp.status == 750) {
                set resp.http.Location = resp.reason;
                set resp.status = 302;
                return(deliver);
        }
}

Edit /etc/default/varnish.
It adds 1Go of memory cache and 10Go of file cache.

DAEMON_OPTS="-a :80 \
             -T localhost:6082 \
             -f /etc/varnish/default.vcl \
             -S /etc/varnish/secret \
	     -s malloc,1G \
	     -s file,/var/cache/varnish/varnish_storage10G.bin,10G"

Restart varnish…

Change your client(s) sources.list:

• Devuan (merged – ascii) – Replace version ‘ascii’ with the one you want

deb http://foo.bar.net/devuan ascii          main contrib non-free
deb http://foo.bar.net/devuan ascii-updates  main contrib non-free
deb http://foo.bar.net/devuan ascii-security main contrib non-free

• Debian (stretch) – Idem with version change

deb http://foo.bar.net/debian stretch         main contrib non-free
deb http://foo.bar.net/debian stretch-updates  main contrib non-free
deb http://foo.bar.net/debian-security stretch-updates main contrib non-free

• Ubuntu (trusty) – Idem with version change

deb http://foo.bar.net/ubuntu/ trusty main restricted
deb http://foo.bar.net/ubuntu/ trusty-updates main restricted

deb http://foo.bar.net/ubuntu/ trusty universe
deb http://foo.bar.net/ubuntu/ trusty-updates universe

deb http://foo.bar.net/ubuntu/ trusty multiverse
deb http://foo.bar.net/ubuntu/ trusty-updates multiverse

deb http://foo.bar.net/ubuntu/ trusty-backports main restricted universe multiverse

deb http://foo.bar.net/ubuntu-security/ trusty-security main restricted
deb http://foo.bar.net/ubuntu-security/ trusty-security universe
deb http://foo.bar.net/ubuntu-security/ trusty-security multiverse