Devuan / Debian cache with varnish

How to add a high performance devuan / debian / ubuntu / cpan repository cache (merged) with the security repo.
Clean access paths:

  • /devuan (merged devuan)
  • /debian (debian)
  • /debian-security (debian security)
  • /ubuntu (ubuntu)
  • /ubuntu-security (ubuntu security)
  • /cpan (bonus)

If none of these paths matches, varnish falls back to the local backend on port 8080. You can add a http server running on localhost:8080 and host your own apt repo like this.

Let’s say your varnish will be accessible via the hostname ‘foo.bar.net’.
You have to replace all occurences of ‘foo.bar.net’ from the configuration with your hostname.

# apt-get install varnish

Edit /etc/varnish/default.vcl:

vcl 4.0;

backend default {
        .host = "127.0.0.1";
        .port = "8080";
}

backend debian {
        .host = "ftp.debian.org";
        .port = "80";
}

backend ubuntu {
        # RR-DNS on us.archive.ubuntu.com... choose one
        .host = "91.189.91.26";
        .port = "80";
}

backend ubuntusec {
        # RR-DNS on security.ubuntu.com... choose one
        .host = "91.189.88.149";
        .port = "80";
}

backend cpan {
        .host = "www.cpan.org";
        .port = "80";
}

backend debiansec {
        # RR-DNS on security.debian.org... choose one
        .host = "217.196.149.233";
        .port = "80";
}

backend devuan {
        .host = "auto.mirror.devuan.org";
        .port = "80";
}


sub vcl_recv {
        if (req.url ~ "^/cpan") {
		set req.backend_hint = cpan;
		set req.url =  regsub( req.url, "^/cpan/", "/");
                set req.http.host = "www.cpan.org";

        } elsif (req.url ~ "^/devuan") {
                if (req.url ~ "^/devuan/pool/DEBIAN-SECURITY") {
                        set req.url = regsub(
                                        req.url,
                                        "^/devuan/pool/DEBIAN-SECURITY/",
                                        "http://foo.bar.net/security/pool/");
                        return ( synth (
                                        750,
                                        req.url
                                       ));
                }
                elsif (req.url ~ "^/devuan/pool/DEBIAN") {
                        set req.url = regsub(
                                        req.url,
                                        "^/devuan/pool/DEBIAN/",
                                        "http://foo.bar.net/debian/pool/");
                        return ( synth (
                                        750,
                                        req.url
                                       ));
                }
                else {
                        set req.url =  regsub(
                                        req.url,
                                        "^/devuan/",
                                        "/merged/");
                }
                set req.backend_hint = devuan;
                set req.http.host = "auto.mirror.devuan.org";

        } elsif (req.url ~ "^/debian/") {
                set req.backend_hint = debian;
                set req.http.host = "ftp.debian.org";

        } elsif (req.url ~ "^/ubuntu/") {
                set req.backend_hint = ubuntu;
                set req.http.host = "us.archive.ubuntu.com";

        } elsif (req.url ~ "^/ubuntu-security/") {
                set req.backend_hint = ubuntusec;
                set req.http.host = "security.ubuntu.com";
                set req.url = regsub( req.url, "^/ubuntu-security/", "/ubuntu/" );

        } elsif (req.url ~ "^/debian-security/") {
                set req.backend_hint = debiansec;
                set req.http.host = "security.debian.org";
                set req.url = regsub( req.url, "-updates/", "/updates/" );

        } elsif (req.url ~ "^/security/") {
                set req.backend_hint = debiansec;
                set req.http.host = "security.debian.org";
                set req.url = regsub( req.url, "^/security/", "/debian-security/" );
                set req.url = regsub( req.url, "-updates/", "/updates/" );
        }
}

sub vcl_backend_response {
        # Don't cache dists
        if (bereq.url ~ "^/(devuan|debian[^/]*|ubuntu[^/]*|merged)/dists") {
                set beresp.ttl = 120s;
                set beresp.uncacheable = true;
                return (deliver);

        } elsif (bereq.url ~ "/modules/.*.gz") {
		# CPAN
		set beresp.ttl = 15s;
		set beresp.uncacheable = true;
		return (deliver);
	}
}

sub vcl_deliver {
}

sub vcl_synth {
        if (resp.status == 750) {
                set resp.http.Location = resp.reason;
                set resp.status = 302;
                return(deliver);
        }
}

Edit /etc/default/varnish.
It adds 1Go of memory cache and 10Go of file cache.

DAEMON_OPTS="-a :80 \
             -T localhost:6082 \
             -f /etc/varnish/default.vcl \
             -S /etc/varnish/secret \
	     -s malloc,1G \
	     -s file,/var/cache/varnish/varnish_storage10G.bin,10G"

Restart varnish…

Change your client(s) sources.list:

  • Devuan (merged – ascii) – Replace version ‘ascii’ with the one you want
  • deb http://foo.bar.net/devuan ascii          main contrib non-free
    deb http://foo.bar.net/devuan ascii-updates  main contrib non-free
    deb http://foo.bar.net/devuan ascii-security main contrib non-free
    
  • Debian (stretch) – Idem with version change
  • deb http://foo.bar.net/debian stretch         main contrib non-free
    deb http://foo.bar.net/debian stretch-updates  main contrib non-free
    deb http://foo.bar.net/debian-security stretch-updates main contrib non-free
    
  • Ubuntu (trusty) – Idem with version change
  • deb http://foo.bar.net/ubuntu/ trusty main restricted
    deb http://foo.bar.net/ubuntu/ trusty-updates main restricted
    
    deb http://foo.bar.net/ubuntu/ trusty universe
    deb http://foo.bar.net/ubuntu/ trusty-updates universe
    
    deb http://foo.bar.net/ubuntu/ trusty multiverse
    deb http://foo.bar.net/ubuntu/ trusty-updates multiverse
    
    deb http://foo.bar.net/ubuntu/ trusty-backports main restricted universe multiverse
    
    deb http://foo.bar.net/ubuntu-security/ trusty-security main restricted
    deb http://foo.bar.net/ubuntu-security/ trusty-security universe
    deb http://foo.bar.net/ubuntu-security/ trusty-security multiverse
    

Leave a Reply

Your email address will not be published. Required fields are marked *