不缓存特定大小的内容

最近使用的一台varnish服务器，经常发现把内存资源和磁盘IO资源占满的情况，后面找原因，发现Varnish把一些大文件缓存了，例如一些 压缩档。记得以前用Squid时是有相应设置项的，可以不缓存文件大小超过1M的文件。经过google搜索了一番，发现有篇文章的方法不错，用变通的方法（vcl_fetch 不支持 pass 语法，但支持 restart）解决了我的困扰，遂写日志以记之。

sub vcl_recv {
  /** ... vcl_recv example from above ... */
  /* Bypass cache for large files.  The x-pipe header is
     set in vcl_fetch when a too large file is detected. */
  if (req.http.x-pipe && req.restarts > 0) {
    remove req.http.x-pipe;
    return (pipe);
  }
  /** ... vcl_recv example from above ... */
}

sub vcl_fetch {
  /* Don't try to cache too large files.  It appears
     Varnish just crashes if we don't filter them. */
  if (beresp.http.Content-Length ~ "[0-9]{7,}") {
    set req.http.x-pipe = "1";
    return (restart);
  }
}

sub vcl_pipe {
  set bereq.http.connection = "close";
}

完整的配置示例：

import std;

backend web1 {
    .host = "172.16.2.31";
    .port = "80";
    .probe = {
        .url = "/";
        .interval = 10s;
        .timeout = 2s;
        .window = 3;
        .threshold = 3;
    }
}
backend web2 {
    .host = "172.16.2.32";
    .port = "80";
    .probe = {
        .url = "/";
        .interval = 10s;
        .timeout = 2s;
        .window = 3;
        .threshold = 3;
    }
}
# 定义负载均衡组
director webgroup random {
    {
       .backend = web1;
       .weight = 1;
    }
    {   
       .backend = web2;
       .weight = 1; 
    }
}

# 允许刷新缓存的ip
acl purgeAllow {
     "localhost";
     "172.16.2.5";
}

sub vcl_recv {
    # 刷新缓存设置
    if (req.request == "PURGE") {
        #判断是否允许ip
        if (!client.ip ~ purgeAllow) {
             error 405 "Not allowed.";
        }
        #去缓存中查找
        return (lookup);
    }

    std.log("LOG_DEBUG: URL=" + req.url); 

    set req.backend = webgroup;
    if (req.request != "GET" && req.request != "HEAD" && req.request != "PUT" && req.request != "POST" && req.request != "TRACE" && req.request != "OPTIONS" && req.request != "DELETE") {
         /* Non-RFC2616 or CONNECT which is weird. */
         return (pipe);
    }

    # 只缓存 GET 和 HEAD 请求
    if (req.request != "GET" && req.request != "HEAD") {
        std.log("LOG_DEBUG: req.request not get!  " + req.request );
        return(pass);
    }
    # http 认证的页面也 pass
　　if (req.http.Authorization) { 
        std.log("LOG_DEBUG: req is authorization !");
        return (pass);
    }
    if (req.http.Cache-Control ~ "no-cache") {
        std.log("LOG_DEBUG: req is no-cache");
        return (pass);
    }
    # 忽略admin、verify、servlet目录，以.jsp和.do结尾以及带有?的URL,直接从后端服务器读取内容
    if (req.url ~ "^/admin" || req.url ~ "^/verify/" || req.url ~ "^/servlet/" || req.url ~ "\.(jsp|php|do)($|\?)") {
        std.log("url is admin or servlet or jsp|php|do, pass.");
        return (pass);
    }

    # 只缓存指定扩展名的请求, 并去除 cookie
    if (req.url ~ "^/[^?]+\.(jpeg|jpg|png|gif|bmp|tif|tiff|ico|wmf|js|css|ejs|swf|txt|zip|exe|html|htm)(\?.*|)$") {
        std.log("*** url is jpeg|jpg|png|gif|ico|js|css|txt|zip|exe|html|htm set cached! ***");
        unset req.http.cookie;
        # 规范请求头,Accept-Encoding 只保留必要的内容
        if (req.http.Accept-Encoding) {
            if (req.url ~ "\.(jpg|png|gif|jpeg)(\?.*|)$") {
                remove req.http.Accept-Encoding;
            } elsif (req.http.Accept-Encoding ~ "gzip") {
                set req.http.Accept-Encoding = "gzip";
            } elsif (req.http.Accept-Encoding ~ "deflate") {
                set req.http.Accept-Encoding = "deflate";
            } else {
                remove req.http.Accept-Encoding;
            }
        }
        return(lookup);
    } else {
        std.log("url is not cached!");
        return (pass);
    }
}

sub vcl_hit {
    if (req.request == "PURGE") {    
        set obj.ttl = 0s;    
        error 200 "Purged.";    
    }
    return (deliver);
}

sub vcl_miss {
    std.log("################# cache miss ################### url=" + req.url);
    if (req.request == "PURGE") {
        purge;
        error 200 "Purged.";
    }
}

sub vcl_fetch {
    # 如果后端服务器返回错误，则进入 saintmode
    if (beresp.status == 500 || beresp.status == 501 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
        std.log("beresp.status error!!! beresp.status=" + beresp.status);
        set req.http.host = "status";
        set beresp.saintmode = 20s;
        return (restart);
    }
    # 如果后端静止缓存, 则跳过
    if (beresp.http.Pragma ~ "no-cache" || beresp.http.Cache-Control ~ "no-cache" || beresp.http.Cache-Control ~ "private") {
        std.log("not allow cached!   beresp.http.Cache-Control=" + beresp.http.Cache-Control);
    return (hit_for_pass);
    }
    if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
        /* Mark as "Hit-For-Pass" for the next 2 minutes */
        set beresp.ttl = 120 s;
        return (hit_for_pass);
    }

    if (req.request == "GET" && req.url ~ "\.(css|js|ejs|html|htm)$") {
        std.log("gzip is enable.");
        set beresp.do_gzip = true;
        set beresp.ttl = 20s;
    }

    if (req.request == "GET" && req.url ~ "^/[^?]+\.(jpeg|jpg|png|gif|bmp|tif|tiff|ico|wmf|js|css|ejs|swf|txt|zip|exe)(\?.*|)$") {
        std.log("url css|js|gif|jpg|jpeg|bmp|png|tiff|tif|ico|swf|exe|zip|bmp|wmf is cache 5m!");
        set beresp.ttl = 5m;
    } elseif (req.request == "GET" && req.url ~ "\.(html|htm)$") {
        set beresp.ttl = 30s;
    } else {
        return (hit_for_pass);
    }
 
    # 如果后端不健康，则先返回缓存数据1分钟
    if (!req.backend.healthy) {
        std.log("eq.backend not healthy! req.grace = 1m");
        set req.grace = 1m;
    } else {
        set req.grace = 30s;
    }
     return (deliver);
}

# 发送给客户端
sub vcl_deliver {
    if ( obj.hits > 0 ) {
    set resp.http.X-Cache = "has cache";
    } else {
    #set resp.http.X-Cache = "no cache";
    }
    return (deliver);
}

# This is a basic VCL configuration file for PageCache powered by Varnish for Magento module.

# default backend definition.  Set this to point to your content server.
backend default {
  .host = "127.0.0.1";
  .port = "8080";
}

# admin backend with longer timeout values. Set this to the same IP & port as your default server.
backend admin {
  .host = "127.0.0.1";
  .port = "8080";
  .first_byte_timeout = 18000s;
  .between_bytes_timeout = 18000s;
}

# add your Magento server IP to allow purges from the backend
acl purge {
  "localhost";
  "127.0.0.1";
}


sub vcl_recv {
    if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
            req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    if (req.request != "GET" &&
      req.request != "HEAD" &&
      req.request != "PUT" &&
      req.request != "POST" &&
      req.request != "TRACE" &&
      req.request != "OPTIONS" &&
      req.request != "DELETE" &&
      req.request != "PURGE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }
    # purge request
    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            error 405 "Not allowed.";
        }
        ban("obj.http.X-Purge-Host ~ " + req.http.X-Purge-Host + " && obj.http.X-Purge-URL ~ " + req.http.X-Purge-Regex + " && obj.http.Content-Type ~ " + req.http.X-Purge-Content-Type);
        error 200 "Purged.";
    }

    # switch to admin backend configuration
    if (req.http.cookie ~ "adminhtml=") {
        set req.backend = admin;
    }
        # if is phpmyadmin no cache it. by samshuai
        if (req.url ~ "oomyadmin\/") {
                 set req.http.connection = "close";
                 return (pipe);
         }
    # we only deal with GET and HEAD by default
    if (req.request != "GET" && req.request != "HEAD") {
        return (pass);
    }

    # normalize url in case of leading HTTP scheme and domain
    set req.url = regsub(req.url, "^http[s]?://[^/]+", "");

    # static files are always cacheable. remove SSL flag and cookie
    if (req.url ~ "^/(media|js|skin)/.*\.(png|jpg|jpeg|gif|css|js|swf|ico)$") {
        unset req.http.Https;
        unset req.http.Cookie;
    }

    # not cacheable by default
    if (req.http.Authorization || req.http.Https) {
        return (pass);
    }

    # do not cache any page from
    # - index files
    # - ...
    if (req.url ~ "^/(index)") {
        return (pass);
    }
    # as soon as we have a NO_CACHE cookie pass request
    if (req.http.cookie ~ "NO_CACHE=") {
        return (pass);
    }

    # normalize Aceept-Encoding header
    # http://varnish.projects.linpro.no/wiki/FAQ/Compression
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
            # No point in compressing these
            remove req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            remove req.http.Accept-Encoding;
        }
    }

    # remove Google gclid parameters
    set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
    set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
    set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"

    return (lookup);
}

# sub vcl_pipe {
#     # Note that only the first request to the backend will have
#     # X-Forwarded-For set.  If you use X-Forwarded-For and want to
#     # have it set for all requests, make sure to have:
#     # set bereq.http.connection = "close";
#     # here.  It is not set by default as it might break some broken web
#     # applications, like iis with NTLM authentication.
#     return (pipe);
# }
#
# sub vcl_pass {
#     return (pass);
# }
#
sub vcl_hash {
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    if (!(req.url ~ "^/(media|js|skin)/.*\.(png|jpg|jpeg|gif|css|js|swf|ico)$")) {
        call design_exception;
    }
    return (hash);
}
#
# sub vcl_hit {
#     return (deliver);
# }
#
# sub vcl_miss {
#     return (fetch);
# }

sub vcl_fetch {
    if (beresp.status == 500) {
       set beresp.saintmode = 10s;
       return (restart);
    }
    set beresp.grace = 5m;

    # add ban-lurker tags to object
    set beresp.http.X-Purge-URL = req.url;
    set beresp.http.X-Purge-Host = req.http.host;

    if (beresp.status == 200 || beresp.status == 301 || beresp.status == 404) {
        if (beresp.http.Content-Type ~ "text/html" || beresp.http.Content-Type ~ "text/xml") {
            if ((beresp.http.Set-Cookie ~ "NO_CACHE=") || (beresp.ttl < 1s)) {
                set beresp.ttl = 0s;
                return (hit_for_pass);
            }

            # marker for vcl_deliver to reset Age:
            set beresp.http.magicmarker = "1";

            # Don't cache cookies
            unset beresp.http.set-cookie;
        } else {
            # set default TTL value for static content
            set beresp.ttl = 4h;
        }
        return (deliver);
    }

    return (hit_for_pass);
}
sub vcl_deliver {
    # debug info
    if (resp.http.X-Cache-Debug) {
        if (obj.hits > 0) {
            set resp.http.X-Cache = "HIT";
            set resp.http.X-Cache-Hits = obj.hits;
        } else {
           set resp.http.X-Cache = "MISS";
        }
        set resp.http.X-Cache-Expires = resp.http.Expires;
    } else {
        # remove Varnish/proxy header
        remove resp.http.X-Varnish;
        remove resp.http.Via;
        remove resp.http.Age;
        remove resp.http.X-Purge-URL;
        remove resp.http.X-Purge-Host;
    }

    if (resp.http.magicmarker) {
        # Remove the magic marker
        unset resp.http.magicmarker;

        set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0";
        set resp.http.Pragma = "no-cache";
        set resp.http.Expires = "Mon, 31 Mar 2008 10:00:00 GMT";
        set resp.http.Age = "0";
    }
}

sub design_exception {
}