I'm trying out Varnish for our Magento eCommerce store and I'm using the Magento Turpentine extension which manages Varnish for Magento,
I have it almost working properly for every page but on the product pages when a logged in user visits a page that has not been cached by Varnish before the resulting html page given back to them has a 403 Forbidden nginx/1.4.6 (Ubuntu) error in the header section of the page, where the logo, navbar etc should be.
.
The issue temporarily goes away when I flush the cache and remove the client cookies but it occurs again.
1. Any thoughts What could be causing this issue?
2. What can I add to my VLC file to fix this? Or even to stop varnish from caching all product pages when a customer is logged in? And as a last resort to never cache them at all?
My full web stack is Ubuntu 14.4, HHVM, NGINX, Varnish & Magento.
UPDATE
I've noticed that if I remove the store cookie from the browser the issue does not occur anymore but then I have the issue that a logged in customer cannot be in the correct store for them.
In my NGINX error log:
$ sudo cat /var/log/nginx/error.log
2015/09/25 10:21:47 [error] 6883#0: *9127 access forbidden by rule, client: 127.0.0.1, server: staging.example.ie, request: "GET /turpentine/esi/getBlock/method/esi/access/private/ttl/7200/hmac/792b5a0b692e443b333d05c67da46b43b2894112a529bc9d491ee1ae6587d643/data/.9SNe-3SWNx-0DG1igFbueN2iD6oX0Px5UAy8UKa6aJeS6iGeL-QzDW518ZNfFbSx.UCkgmgsQudLS-7Yw7xlvFq8dRYYHyzvSnedsQr-SIdjJOc6qPPCZDz.thRPlO55ZbfrbyajL10WOM9hY.sW0aISLHNaMh-zKoxa3coOgK.RaVwdxwDQNko-kLgBFCpCi5dlKjeqtwCnbCtxaVlUS5TxWUlMj0x8SwmG3TAsj-dZ6wPu-IqERrZCfx12HzY2EXV4tP04qlcGsvrgKYt91dE-10NL15A2zICMeISM9JCzX-W8n9nyEVGxdedFpDUKBvqM7O3-cTzHs-3ZKCagKOSudNe3eiBaHBFX2yf.vWAlc-E0fr6-NsPuwoHxwCQcdTjC22n4z0Cab8Y5IQI09bAYZXVH0bUOsOVG7n7utY=/ HTTP/1.1", host: "staging.example.ie", referrer: "http://staging.example.ie/kiddies-scarf"
These are the page headers
Remote Address:52.7.40.70:80
Request URL:http://staging.example.ie/lace-tumbler
Request Method:GET
Status Code:200 OK
Response Headers
view source
Age:14
Connection:keep-alive
Content-Encoding:gzip
Content-Type:text/html; charset=UTF-8
Date:Mon, 28 Sep 2015 10:00:49 GMT
Transfer-Encoding:chunked
X-Frame-Options:SAMEORIGIN
Request Headers
view source
Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
Accept-Encoding:gzip, deflate, sdch
Accept-Language:en-US,en;q=0.8,sv;q=0.6
Cache-Control:no-cache
Connection:keep-alive
Cookie:store=ie; currency=EUR; cookielaw=1; frontend=1185f1d3446c7e158bdb4e73572fcbf0; customer_group=1; frontend=1185f1d3446c7e158bdb4e73572fcbf0; external_no_cache=1
Host:staging.example.ie
Pragma:no-cache
Referer:http://staging.example.ie/crystal
Upgrade-Insecure-Requests:1
User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.99 Safari/537.36
In case you're wondering were some of the Varnish headers are, the Turpentine extension removes them.
VCL File
C{
#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <pthread.h>
static pthread_mutex_t lrand_mutex = PTHREAD_MUTEX_INITIALIZER;
void generate_uuid(char* buf) {
pthread_mutex_lock(&lrand_mutex);
long a = lrand48();
long b = lrand48();
long c = lrand48();
long d = lrand48();
pthread_mutex_unlock(&lrand_mutex);
sprintf(buf, "frontend=%08lx%04lx%04lx%04lx%04lx%08lx",
a,
b & 0xffff,
(b & ((long)0x0fff0000) >> 16) | 0x4000,
(c & 0x0fff) | 0x8000,
(c & (long)0xffff0000) >> 16,
d
);
return;
}
}C
import std;
backend default {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 300s;
.between_bytes_timeout = 300s;
}
backend admin {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 21600s;
.between_bytes_timeout = 21600s;
}
acl crawler_acl {
"127.0.0.1";
}
acl debug_acl {
}
sub generate_session {
if (req.url ~ ".*[&?]SID=([^&]+).*") {
set req.http.X-Varnish-Faked-Session = regsub(
req.url, ".*[&?]SID=([^&]+).*", "frontend=\1");
} else {
C{
char uuid_buf [50];
generate_uuid(uuid_buf);
VRT_SetHdr(sp, HDR_REQ,
"\030X-Varnish-Faked-Session:",
uuid_buf,
vrt_magic_string_end
);
}C
}
if (req.http.Cookie) {
std.collect(req.http.Cookie);
set req.http.Cookie = req.http.X-Varnish-Faked-Session +
"; " + req.http.Cookie;
} else {
set req.http.Cookie = req.http.X-Varnish-Faked-Session;
}
}
sub generate_session_expires {
C{
time_t now = time(NULL);
struct tm now_tm = *gmtime(&now);
now_tm.tm_sec += 7200;
mktime(&now_tm);
char date_buf [50];
strftime(date_buf, sizeof(date_buf)-1, "%a, %d-%b-%Y %H:%M:%S %Z", &now_tm);
VRT_SetHdr(sp, HDR_RESP,
"\031X-Varnish-Cookie-Expires:",
date_buf,
vrt_magic_string_end
);
}C
}
sub vcl_recv {
if (req.restarts == 0) {
if (req.http.X-Forwarded-For) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
if (req.http.Accept-Encoding) {
if (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} else if (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
if (req.http.User-Agent ~ "iP(?:hone|ad|od)|BlackBerry|Palm|Googlebot-Mobile|Mobile|mobile|mobi|Windows Mobile|Safari Mobile|Android|Opera (?:Mini|Mobi)") {
set req.http.X-Normalized-User-Agent = "mobile";
} else if (req.http.User-Agent ~ "MSIE") {
set req.http.X-Normalized-User-Agent = "msie";
} else if (req.http.User-Agent ~ "Firefox") {
set req.http.X-Normalized-User-Agent = "firefox";
} else if (req.http.User-Agent ~ "Chrome") {
set req.http.X-Normalized-User-Agent = "chrome";
} else if (req.http.User-Agent ~ "Safari") {
set req.http.X-Normalized-User-Agent = "safari";
} else if (req.http.User-Agent ~ "Opera") {
set req.http.X-Normalized-User-Agent = "opera";
} else {
set req.http.X-Normalized-User-Agent = "other";
}
if (!true || req.http.Authorization ||
req.request !~ "^(GET|HEAD|OPTIONS)$" ||
req.http.Cookie ~ "varnish_bypass=1") {
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?admin") {
set req.backend = admin;
}
return (pipe);
}
set req.url = regsuball(req.url, "([^:])//+", "\1/");
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
set req.http.X-Turpentine-Secret-Handshake = "1";
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?admin") {
set req.backend = admin;
return (pipe);
}
if (req.http.Cookie ~ "\bcurrency=") {
set req.http.X-Varnish-Currency = regsub(
req.http.Cookie, ".*\bcurrency=([^;]*).*", "\1");
}
if (req.http.Cookie ~ "\bstore=") {
set req.http.X-Varnish-Store = regsub(
req.http.Cookie, ".*\bstore=([^;]*).*", "\1");
}
if (req.url ~ "/turpentine/esi/get(?:Block|FormKey)/") {
set req.http.X-Varnish-Esi-Method = regsub(
req.url, ".*/method/(\w+)/.*", "\1");
set req.http.X-Varnish-Esi-Access = regsub(
req.url, ".*/access/(\w+)/.*", "\1");
if (req.http.X-Varnish-Esi-Method == "esi" && req.esi_level == 0 &&
!(false || client.ip ~ debug_acl)) {
error 403 "External ESI requests are not allowed";
}
}
if (req.http.Cookie !~ "frontend=" && !req.http.X-Varnish-Esi-Method) {
if (client.ip ~ crawler_acl ||
req.http.User-Agent ~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*Siege.*|magespeedtest\.com|Nexcessnet_Turpentine/.*)$") {
set req.http.Cookie = "frontend=crawler-session";
} else {
call generate_session;# call generate_session;
return (pipe);
}
}
if (true &&
req.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
unset req.http.Cookie;
unset req.http.X-Varnish-Faked-Session;
return (lookup);
}
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?(?:admin|api|cron\.php|checkout|checkout/cart|customer|advancednewsletter|onestepcheckout|onepage|exporter|contacts|iphone|join|sitemap.xml)" ||
req.url ~ "\?.*__from_store=") {
return (pipe);
}
if (true &&
req.url ~ "(?:[?&](?:__SID|XDEBUG_PROFILE)(?=[&=]|$))") {
return (pass);
}
if (true && req.url ~ "[?&](utm_source|utm_medium|utm_campaign|utm_content|utm_term|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|utm_content|utm_term|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
return (lookup);
}
}
sub vcl_pipe {
unset bereq.http.X-Turpentine-Secret-Handshake;
set bereq.http.Connection = "close";
}
sub vcl_hash {
hash_data(req.url);
if (req.http.Host) {
hash_data(req.http.Host);
} else {
hash_data(server.ip);
}
hash_data(req.http.Ssl-Offloaded);
if (req.http.X-Normalized-User-Agent) {
hash_data(req.http.X-Normalized-User-Agent);
}
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
}
if (req.http.X-Varnish-Store || req.http.X-Varnish-Currency) {
hash_data("s=" + req.http.X-Varnish-Store + "&c=" + req.http.X-Varnish-Currency);
}
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
hash_data(regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
}
if (req.http.X-Varnish-Esi-Access == "customer_group" &&
req.http.Cookie ~ "customer_group=") {
hash_data(regsub(req.http.Cookie, "^.*?customer_group=([^;]*);*.*$", "\1"));
}
return (hash);
}
sub vcl_hit {
}
sub vcl_fetch {
set req.grace = 15s;
set beresp.http.X-Varnish-Host = req.http.host;
set beresp.http.X-Varnish-URL = req.url;
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
unset beresp.http.Vary;
set beresp.do_gzip = true;
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 15s;
return (hit_for_pass);
} else {
if (beresp.http.Set-Cookie) {
set beresp.http.X-Varnish-Set-Cookie = beresp.http.Set-Cookie;
unset beresp.http.Set-Cookie;
}
unset beresp.http.Cache-Control;
unset beresp.http.Expires;
unset beresp.http.Pragma;
unset beresp.http.Cache;
unset beresp.http.Age;
if (beresp.http.X-Turpentine-Esi == "1") {
set beresp.do_esi = true;
}
if (beresp.http.X-Turpentine-Cache == "0") {
set beresp.ttl = 15s;
return (hit_for_pass);
} else {
if (true &&
bereq.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
set beresp.ttl = 604800s;
set beresp.http.Cache-Control = "max-age=604800";
} elseif (req.http.X-Varnish-Esi-Method) {
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
set beresp.http.X-Varnish-Session = regsub(req.http.Cookie,
"^.*?frontend=([^;]*);*.*$", "\1");
}
if (req.http.X-Varnish-Esi-Method == "ajax" &&
req.http.X-Varnish-Esi-Access == "public") {
set beresp.http.Cache-Control = "max-age=" + regsub(
req.url, ".*/ttl/(\d+)/.*", "\1");
}
set beresp.ttl = std.duration(
regsub(
req.url, ".*/ttl/(\d+)/.*", "\1s"),
300s);
if (beresp.ttl == 0s) {
set beresp.ttl = 15s;
return (hit_for_pass);
}
} else {
set beresp.ttl = 7200s;
}
}
}
return (deliver);
}
}
sub vcl_deliver {
if (req.http.X-Varnish-Faked-Session) {
call generate_session_expires;
set resp.http.Set-Cookie = req.http.X-Varnish-Faked-Session +
"; expires=" + resp.http.X-Varnish-Cookie-Expires + "; path=/";
if (req.http.Host) {
if (req.http.User-Agent ~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*Siege.*|magespeedtest\.com|Nexcessnet_Turpentine/.*)$") {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
} else {
if(req.http.Host ~ "") {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=";
} else {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
}
}
}
set resp.http.Set-Cookie = resp.http.Set-Cookie + "; httponly";
unset resp.http.X-Varnish-Cookie-Expires;
}
if (req.http.X-Varnish-Esi-Method == "ajax" && req.http.X-Varnish-Esi-Access == "private") {
set resp.http.Cache-Control = "no-cache";
}
if (false || client.ip ~ debug_acl) {
set resp.http.X-Varnish-Hits = obj.hits;
set resp.http.X-Varnish-Esi-Method = req.http.X-Varnish-Esi-Method;
set resp.http.X-Varnish-Esi-Access = req.http.X-Varnish-Esi-Access;
set resp.http.X-Varnish-Currency = req.http.X-Varnish-Currency;
set resp.http.X-Varnish-Store = req.http.X-Varnish-Store;
} else {
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Turpentine-Cache;
unset resp.http.X-Turpentine-Esi;
unset resp.http.X-Turpentine-Flush-Events;
unset resp.http.X-Turpentine-Block;
unset resp.http.X-Varnish-Session;
unset resp.http.X-Varnish-Host;
unset resp.http.X-Varnish-URL;
unset resp.http.X-Varnish-Set-Cookie;
}
}
NGINX sites-available/default-staging configuration file
server {
# Listen on port 80 as well as post 443 for SSL connections.
listen 8080;
#listen 443 default ssl;
server_name staging.example.ie;
# Specify path to your SSL certificates.
#ssl_certificate /etc/nginx/certificates/yourcertificate.crt;
#ssl_certificate_key /etc/nginx/certificates/yourcertificate.key;
root /var/www/staging-hoi/;
#auth_basic "Restricted website - authorised access only";
#auth_basic_user_file /etc/nginx/.htpasswd;
location / {
index index.html index.htm index.php;
try_files $uri $uri/ @handler;
}
# Deny access to specific directories no one
# in particular needs access to anyways.
location /app/ { deny all; }
location /includes/ { deny all; }
location /lib/ { deny all; }
location /media/downloadable/ { deny all; }
location /pkginfo/ { deny all; }
location /report/config.xml { deny all; }
location /var/ { deny all; }
# Allow only those who have a login name and password
# to view the export folder. Refer to /etc/nginx/htpassword.
#location /var/export/ {
# auth_basic "Restricted";
# auth_basic_user_file htpasswd;
# autoindex on;
#}
# Deny all attempts to access hidden files
# such as .htaccess, .htpasswd, etc...
location ~ /\. {
deny all;
access_log off;
log_not_found off;
}
# This redirect is added so to use Magentos
# common front handler when handling incoming URLs.
location @handler {
rewrite / /index.php;
}
# Forward paths such as /js/index.php/x.js
# to their relevant handler.
location ~ .php/ {
rewrite ^(.*.php)/ $1 last;
}
##
# Rewrite for versioned CSS+JS via filemtime
##
location ~* ^.+\.(css|js)$ {
rewrite ^(.+)\.(\d+)\.(css|js)$ $1.$3 last;
expires 31536000s;
access_log off;
log_not_found off;
add_header Pragma public;
add_header Cache-Control "max-age=31536000, public";
}
##
# Aggressive caching for static files
# If you alter static files often, please use
# add_header Cache-Control "max-age=31536000, public, must-revalidate, proxy-revalidate";
##
location ~* \.(asf|asx|wax|wmv|wmx|avi|bmp|class|divx|doc|docx|eot|exe|gif|gz|gzip|ico|jpg|jpeg|jpe|mdb|mid|midi|mov|qt|mp3|m4a|mp4|m4v|mpeg|mpg|mpe|mpp|odb|odc|odf|odg|odp|ods|odt|ogg|ogv|otf|pdf|png|pot|pps|ppt|pptx|ra|ram|svg|svgz|swf|tar|t?gz|tif|tiff|ttf|wav|webm|wma|woff|wri|xla|xls|xlsx|xlt|xlw|zip)$ {
expires 31536000s;
access_log off;
log_not_found off;
add_header Pragma public;
add_header Cache-Control "max-age=31536000, public";
}
# Handle the exectution of .php files.
location ~ .php$ {
if (!-e $request_filename) {
rewrite / /index.php last;
}
expires off;
# --PHP5-FPM CONFIG START (keep fastcgi_param HTTPS OFF)--
#fastcgi_pass unix:/var/run/php5-fpm.sock;
##fastcgi_param HTTPS $fastcgi_https;
#fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
# --PHP5-FPM CONFIG START--
# --HHVM CONFIG START--
fastcgi_pass 127.0.0.1:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
try_files $uri $uri/ @handler;
# --HHVM CONFIG END--
fastcgi_param MAGE_RUN_CODE default;
fastcgi_param MAGE_RUN_TYPE store;
include fastcgi_params;
}
}
My /etc/nginx/nginx.conf is pretty much at it's default settings, I can share this if you need.
