HTTP/1.1 503 Backend fetch failed varnish 6.5.1

[legend]

Curl my webiste and i get the following error:

HTTP/1.1 503 Backend fetch failed
< Server: nginx/1.18.0
Server: nginx/1.18.0
< Date: Sun, 29 Nov 2020 18:26:44 GMT
Date: Sun, 29 Nov 2020 18:26:44 GMT
< Content-Type: text/html; charset=utf-8
Content-Type: text/html; charset=utf-8
< Content-Length: 278
Content-Length: 278
< Connection: keep-alive
Connection: keep-alive
< Keep-Alive: timeout=60
Keep-Alive: timeout=60
< Retry-After: 5
Retry-After: 5
< Pragma: no-cache
Pragma: no-cache
< Expires: -1
Expires: -1
< Cache-Control: no-store, no-cache, must-revalidate, max-age=0
Cache-Control: no-store, no-cache, must-revalidate, max-age=0

<

• Connection #0 to host bezateli.com left intact

**varnishlog -d -q "BerespStatus == 503" **:

<< BeReq >> 3

• Begin bereq 2 fetch
• VCL_use boot
• Timestamp Start: 1606668463.352764 0.000000 0.000000
• BereqMethod HEAD
• BereqURL /
• BereqProtocol HTTP/1.0
• BereqHeader Host: bezateli.com
• BereqHeader X-Real-IP: 208.77.22.99
• BereqHeader X-Forwarded-Host: bezateli.com
• BereqHeader X-Forwarded-Server: bezateli.com
• BereqHeader X-Forwarded-Proto: http
• BereqHeader User-Agent: curl/7.29.0
• BereqHeader Accept: /
• BereqHeader X-Forwarded-For: 208.77.22.99, 208.77.22.99
• BereqHeader grace: none
• BereqMethod GET
• BereqHeader Accept-Encoding: gzip
• BereqProtocol HTTP/1.1
• BereqHeader X-Varnish: 3
• VCL_call BACKEND_FETCH
• VCL_return fetch
• FetchError backend default: unhealthy
• Timestamp Beresp: 1606668463.352832 0.000068 0.000068
• Timestamp Error: 1606668463.352835 0.000071 0.000002
• BerespProtocol HTTP/1.1
• BerespStatus 503
• BerespReason Backend fetch failed
• BerespHeader Date: Sun, 29 Nov 2020 16:47:43 GMT
• BerespHeader Server: Varnish
• VCL_call BACKEND_ERROR
• BerespHeader content-type: text/html; charset=utf-8
• BerespHeader Retry-After: 5
• VCL_return deliver
• Storage malloc Transient
• Length 278
• BereqAcct 0 0 0 0 0 0
• End

My default.vcl generated from magento:

VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 6

vcl 4.0;

import std;

The minimal Varnish version is 6.0

For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'

backend default {
.host = "myIP";
.port = "8181";
.first_byte_timeout = 600s;
.probe = {

.url = "/health_check.php";

    .url = "/pub/health_check.php";
    .timeout = 2s;
    .interval = 5s;
    .window = 10;
    .threshold = 5;

}
}
acl purge {
"bezateli.com";
}

sub vcl_recv {
if (req.restarts > 0) {
set req.hash_always_miss = true;
}

if (req.method == "PURGE") {
    if (client.ip !~ purge) {
        return (synth(405, "Method not allowed"));
    }
    # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
    # has been added to the response in your backend server config. This is used, for example, by the
    # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
    if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
        return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
    }
    if (req.http.X-Magento-Tags-Pattern) {

ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
}
if (req.http.X-Pool) {
ban("obj.http.X-Pool ~ " + req.http.X-Pool);
}
return (synth(200, "Purged"));
}

if (req.method != "GET" &&
    req.method != "HEAD" &&
    req.method != "PUT" &&
    req.method != "POST" &&
    req.method != "TRACE" &&
    req.method != "OPTIONS" &&
    req.method != "DELETE") {
      /* Non-RFC2616 or CONNECT which is weird. */
      return (pipe);
}

# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}

# Bypass shopping cart, checkout and search requests
if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
    return (pass);
}

# Bypass health check requests
if (req.url ~ "/pub/health_check.php") {
    return (pass);
}

# Set initial grace period usage status
set req.http.grace = "none";

# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");

# collect all cookies
std.collect(req.http.Cookie);

# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
        # No point in compressing these
        unset req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
        set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
        set req.http.Accept-Encoding = "deflate";
    } else {
        # unknown algorithm
        unset req.http.Accept-Encoding;
    }
}

# Remove all marketing get parameters to minimize the cache objects
if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
  set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
    set req.url = regsub(req.url, "[?|&]+$", "");
}

# Static files caching
if (req.url ~ "^/(pub/)?(media|static)/") {
    # Static files should not be cached by default
    return (pass);

    # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
    #unset req.http.Https;
    #unset req.http.X-Forwarded-Proto;
    #unset req.http.Cookie;
}

# Authenticated GraphQL requests should not be cached by default
if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") {
    return (pass);
}

return (hash);

}

sub vcl_hash {
if (req.http.cookie ~ "X-Magento-Vary=") {
hash_data(regsub(req.http.cookie, "^.?X-Magento-Vary=([^;]+);.*$", "\1"));
}

# For multi site configurations to not cache each other's content
if (req.http.host) {
    hash_data(req.http.host);
} else {
    hash_data(server.ip);
}

# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
    hash_data(req.http.X-Forwarded-Proto);
}


if (req.url ~ "/graphql") {
    call process_graphql_headers;
}

}

sub process_graphql_headers {
if (req.http.Store) {
hash_data(req.http.Store);
}
if (req.http.Content-Currency) {
hash_data(req.http.Content-Currency);
}
}

sub vcl_backend_response {

set beresp.grace = 3d;

if (beresp.http.content-type ~ "text") {
    set beresp.do_esi = true;
}

if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
    set beresp.do_gzip = true;
}

if (beresp.http.X-Magento-Debug) {
    set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}

# cache only successfully responses and 404s
if (beresp.status != 200 && beresp.status != 404) {
    set beresp.ttl = 0s;
    set beresp.uncacheable = true;
    return (deliver);
} elsif (beresp.http.Cache-Control ~ "private") {
    set beresp.uncacheable = true;
    set beresp.ttl = 86400s;
    return (deliver);
}

# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
    unset beresp.http.set-cookie;
}

If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass

if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store") ||
beresp.http.Vary == "*") {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}

return (deliver);

}

sub vcl_deliver {
if (resp.http.X-Magento-Debug) {
if (resp.http.x-varnish ~ " ") {
set resp.http.X-Magento-Cache-Debug = "HIT";
set resp.http.Grace = req.http.grace;
} else {
set resp.http.X-Magento-Cache-Debug = "MISS";
}
} else {
unset resp.http.Age;
}

# Not letting browser to cache non-static files.
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
    set resp.http.Pragma = "no-cache";
    set resp.http.Expires = "-1";
    set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";

}

unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;

}

sub vcl_hit {
if (obj.ttl >= 0s) {
# Hit within TTL period
return (deliver);
}
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 300s > 0s) {
# Hit after TTL expiration, but within grace period
set req.http.grace = "normal (healthy server)";
return (deliver);
} else {
# Hit after TTL and grace expiration
return (restart);
}
} else {
# server is not healthy, retrieve from cache
set req.http.grace = "unlimited (unhealthy server)";
return (deliver);
}
}

**systemctl edit --full varnish: **

ExecStart=/usr/sbin/varnishd
-a :6081
-a :82
-p feature=+http2
-f /etc/varnish/default.vcl
-s malloc,1024m
-p thread_pool_max=1500
-p pipe_timeout=600
-p http_resp_hdr_len=65536
-p http_resp_size=98304
ExecReload=/usr/sbin/varnishreload

1. When I comment out the probe statement (not recommended from what I read) in the default.vcl, the magento page runs find with no errors.

How do i solve this backend error?

Cheers.

[sandeep]

@legend this can't be done via support forum you need to check this with the developer they should make varnish vcl file which need to support vhosts/sni system.

alternatively you can try php redis cache with magento which is way better than varnish.