HTTPCache.cc
// -*- mode: c++; c-basic-offset:4 -*-
// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.
// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
#include "config.h"
// #define DODS_DEBUG
// #define DODS_DEBUG2
#undef USE_GETENV
#include <pthread.h>
#include <limits.h>
#include <unistd.h> // for stat
#include <sys/types.h> // for stat and mkdir
#include <sys/stat.h>
#include <cstring>
#include <cerrno>
#include <iostream>
#include <sstream>
#include <algorithm>
#include <iterator>
#include <set>
#include "Error.h"
#include "InternalErr.h"
#include "ResponseTooBigErr.h"
#ifndef WIN32
#include "SignalHandler.h"
#endif
#include "HTTPCacheInterruptHandler.h"
#include "HTTPCacheTable.h"
#include "HTTPCache.h"
#include "HTTPCacheMacros.h"
#include "SignalHandlerRegisteredErr.h"
#include "util_mit.h"
#include "debug.h"
using namespace std;
namespace libdap {
HTTPCache *HTTPCache::_instance = 0;
// instance_mutex is used to ensure that only one instance is created.
// That is, it protects the body of the HTTPCache::instance() method. This
// mutex is initialized from within the static function once_init_routine()
// and the call to that takes place using pthread_once_init() where the mutex
// once_block is used to protect that call. All of this ensures that no matter
// how many threads call the instance() method, only one instance is ever
// made.
static pthread_mutex_t instance_mutex;
static pthread_once_t once_block = PTHREAD_ONCE_INIT;
#define NO_LM_EXPIRATION 24*3600 // 24 hours
#define DUMP_FREQUENCY 10 // Dump index every x loads
#define MEGA 0x100000L
#define CACHE_TOTAL_SIZE 20 // Default cache size is 20M
#define CACHE_FOLDER_PCT 10 // 10% of cache size for metainfo etc.
#define CACHE_GC_PCT 10 // 10% of cache size free after GC
#define MIN_CACHE_TOTAL_SIZE 5 // 5M Min cache size
#define MAX_CACHE_ENTRY_SIZE 3 // 3M Max size of single cached entry
static void
once_init_routine()
{
int status;
status = INIT(&instance_mutex);
if (status != 0)
throw InternalErr(__FILE__, __LINE__, "Could not initialize the HTTP Cache mutex. Exiting.");
}
HTTPCache *
HTTPCache::instance(const string &cache_root, bool force)
{
int status = pthread_once(&once_block, once_init_routine);
if (status != 0)
throw InternalErr(__FILE__, __LINE__, "Could not initialize the HTTP Cache mutex. Exiting.");
LOCK(&instance_mutex);
DBG(cerr << "Entering instance(); (" << hex << _instance << dec << ")" << "... ");
try {
if (!_instance) {
_instance = new HTTPCache(cache_root, force);
DBG(cerr << "New instance: " << _instance << ", cache root: "
<< _instance->d_cache_root << endl);
atexit(delete_instance);
#ifndef WIN32
// Register the interrupt handler. If we've already registered
// one, barf. If this becomes a problem, hack SignalHandler so
// that we can chain these handlers... 02/10/04 jhrg
//
// Technically we're leaking memory here. However, since this
// class is a singleton, we know that only three objects will
// ever be created and they will all exist until the process
// exits. We can let this slide... 02/12/04 jhrg
EventHandler *old_eh = SignalHandler::instance()->register_handler(SIGINT, new HTTPCacheInterruptHandler, true);
if (old_eh) {
SignalHandler::instance()->register_handler(SIGINT, old_eh);
throw SignalHandlerRegisteredErr(
"Could not register event handler for SIGINT without superseding an existing one.");
}
old_eh = SignalHandler::instance()->register_handler(SIGPIPE, new HTTPCacheInterruptHandler, true);
if (old_eh) {
SignalHandler::instance()->register_handler(SIGPIPE, old_eh);
throw SignalHandlerRegisteredErr(
"Could not register event handler for SIGPIPE without superseding an existing one.");
}
old_eh = SignalHandler::instance()->register_handler(SIGTERM, new HTTPCacheInterruptHandler, true);
if (old_eh) {
SignalHandler::instance()->register_handler(SIGTERM, old_eh);
throw SignalHandlerRegisteredErr(
"Could not register event handler for SIGTERM without superseding an existing one.");
}
#endif
}
}
catch (...) {
DBG2(cerr << "The constructor threw an Error!" << endl);
UNLOCK(&instance_mutex);
throw;
}
UNLOCK(&instance_mutex);
DBGN(cerr << "returning " << hex << _instance << dec << endl);
return _instance;
}
void
HTTPCache::delete_instance()
{
DBG(cerr << "Entering delete_instance()..." << endl);
if (HTTPCache::_instance) {
DBG(cerr << "Deleting the cache: " << HTTPCache::_instance << endl);
delete HTTPCache::_instance;
HTTPCache::_instance = 0;
//Now remove the signal handlers
delete SignalHandler::instance()->remove_handler(SIGINT);
delete SignalHandler::instance()->remove_handler(SIGPIPE);
delete SignalHandler::instance()->remove_handler(SIGTERM);
}
DBG(cerr << "Exiting delete_instance()" << endl);
}
HTTPCache::HTTPCache(string cache_root, bool force) :
d_locked_open_file(0),
d_cache_enabled(false),
d_cache_protected(false),
d_cache_disconnected(DISCONNECT_NONE),
d_expire_ignored(false),
d_always_validate(false),
d_total_size(CACHE_TOTAL_SIZE * MEGA),
d_folder_size(CACHE_TOTAL_SIZE / CACHE_FOLDER_PCT),
d_gc_buffer(CACHE_TOTAL_SIZE / CACHE_GC_PCT),
d_max_entry_size(MAX_CACHE_ENTRY_SIZE * MEGA),
d_default_expiration(NO_LM_EXPIRATION),
d_max_age(-1),
d_max_stale(-1),
d_min_fresh(-1),
d_http_cache_table(0)
{
DBG(cerr << "Entering the constructor for " << this << "... ");
#if 0
int status = pthread_once(&once_block, once_init_routine);
if (status != 0)
throw InternalErr(__FILE__, __LINE__, "Could not initialize the HTTP Cache mutex. Exiting.");
#endif
INIT(&d_cache_mutex);
// This used to throw an Error object if we could not get the
// single user lock. However, that results in an invalid object. It's
// better to have an instance that has default values. If we cannot get
// the lock, make sure to set the cache as *disabled*. 03/12/03 jhrg
//
// I fixed this block so that the cache root is set before we try to get
// the single user lock. That was the fix for bug #661. To make that
// work, I had to move the call to create_cache_root out of
// set_cache_root(). 09/08/03 jhrg
set_cache_root(cache_root);
int block_size;
if (!get_single_user_lock(force))
throw Error(internal_error, "Could not get single user lock for the cache");
#ifdef WIN32
// Windows is unable to provide us this information. 4096 appears
// a best guess. It is likely to be in the range [2048, 8192] on
// windows, but will the level of truth of that statement vary over
// time ?
block_size = 4096;
#else
struct stat s;
if (stat(cache_root.c_str(), &s) == 0)
block_size = s.st_blksize;
else
throw Error(internal_error, "Could not set file system block size.");
#endif
d_http_cache_table = new HTTPCacheTable(d_cache_root, block_size);
d_cache_enabled = true;
DBGN(cerr << "exiting" << endl);
}
HTTPCache::~HTTPCache()
{
DBG(cerr << "Entering the destructor for " << this << "... ");
try {
if (startGC())
perform_garbage_collection();
d_http_cache_table->cache_index_write();
}
catch (Error &e) {
// If the cache index cannot be written, we've got problems. However,
// unless we're debugging, still free up the cache table in memory.
// How should we let users know they cache index is not being
// written?? 10/03/02 jhrg
DBG(cerr << e.get_error_message() << endl);
}
delete d_http_cache_table;
release_single_user_lock();
DBGN(cerr << "exiting destructor." << endl);
DESTROY(&d_cache_mutex);
}
bool
HTTPCache::stopGC() const
{
return (d_http_cache_table->get_current_size() + d_folder_size < d_total_size - d_gc_buffer);
}
bool
HTTPCache::startGC() const
{
DBG(cerr << "startGC, current_size: " << d_http_cache_table->get_current_size() << endl);
return (d_http_cache_table->get_current_size() + d_folder_size > d_total_size);
}
void
HTTPCache::perform_garbage_collection()
{
DBG(cerr << "Performing garbage collection" << endl);
// Remove all the expired responses.
expired_gc();
// Remove entries larger than max_entry_size.
too_big_gc();
// Remove entries starting with zero hits, 1, ..., until stopGC()
// returns true.
hits_gc();
}
void
HTTPCache::expired_gc()
{
if (!d_expire_ignored) {
d_http_cache_table->delete_expired_entries();
}
}
void
HTTPCache::hits_gc()
{
int hits = 0;
if (startGC()) {
while (!stopGC()) {
d_http_cache_table->delete_by_hits(hits);
hits++;
}
}
}
void HTTPCache::too_big_gc() {
if (startGC())
d_http_cache_table->delete_by_size(d_max_entry_size);
}
bool HTTPCache::get_single_user_lock(bool force)
{
if (!d_locked_open_file) {
FILE * fp = NULL;
try {
// It's OK to call create_cache_root if the directory already
// exists.
create_cache_root(d_cache_root);
}
catch (Error &e) {
// We need to catch and return false because this method is
// called from a ctor and throwing at this point will result in a
// partially constructed object. 01/22/04 jhrg
DBG(cerr << "Failure to create the cache root" << endl);
return false;
}
// Try to read the lock file. If we can open for reading, it exists.
string lock = d_cache_root + CACHE_LOCK;
if ((fp = fopen(lock.c_str(), "r")) != NULL) {
int res = fclose(fp);
if (res) {
DBG(cerr << "Failed to close " << (void *)fp << endl);
}
if (force)
REMOVE(lock.c_str());
else
return false;
}
if ((fp = fopen(lock.c_str(), "w")) == NULL) {
DBG(cerr << "Could not open for write access" << endl);
return false;
}
d_locked_open_file = fp;
return true;
}
DBG(cerr << "locked_open_file is true" << endl);
return false;
}
void
HTTPCache::release_single_user_lock()
{
if (d_locked_open_file) {
int res = fclose(d_locked_open_file);
if (res) {
DBG(cerr << "Failed to close " << (void *)d_locked_open_file << endl) ;
}
d_locked_open_file = 0;
}
string lock = d_cache_root + CACHE_LOCK;
REMOVE(lock.c_str());
}
string
HTTPCache::get_cache_root() const
{
return d_cache_root;
}
void
HTTPCache::create_cache_root(const string &cache_root)
{
#ifdef WIN32
string::size_type cur = cache_root[1] == ':' ? 3 : 1;
typedef int mode_t;
while ((cur = cache_root.find(DIR_SEPARATOR_CHAR, cur)) != string::npos) {
string dir = cache_root.substr(0, cur);
struct stat stat_info;
if (stat(dir.c_str(), &stat_info) == -1) {
DBG2(cerr << "Cache....... Creating " << dir << endl);
mode_t mask = UMASK(0);
if (MKDIR(dir.c_str(), 0777) < 0) {
DBG2(cerr << "Error: can't create." << endl);
UMASK(mask);
throw Error(string("Could not create the directory for the cache. Failed when building path at ") + dir + string("."));
}
UMASK(mask);
}
else {
DBG2(cerr << "Cache....... Found " << dir << endl);
}
cur++;
}
#else
// OSX and Linux
// Save the mask
mode_t mask = umask(0);
// Ignore the error if the directory exists
errno = 0;
if (mkdir(cache_root.c_str(), 0777) < 0 && errno != EEXIST) {
umask(mask);
throw Error("Could not create the directory for the cache at '" + cache_root + "' (" + strerror(errno) + ").");
}
// Restore themask
umask(mask);
#endif
}
void
HTTPCache::set_cache_root(const string &root)
{
if (root != "") {
d_cache_root = root;
// cache root should end in /.
if (d_cache_root[d_cache_root.size()-1] != DIR_SEPARATOR_CHAR)
d_cache_root += DIR_SEPARATOR_CHAR;
}
else {
// If no cache root has been indicated then look for a suitable
// location.
#ifdef USE_GETENV
char * cr = (char *) getenv("DODS_CACHE");
if (!cr) cr = (char *) getenv("TMP");
if (!cr) cr = (char *) getenv("TEMP");
if (!cr) cr = (char*)CACHE_LOCATION;
d_cache_root = cr;
#else
d_cache_root = CACHE_LOCATION;
#endif
if (d_cache_root[d_cache_root.size()-1] != DIR_SEPARATOR_CHAR)
d_cache_root += DIR_SEPARATOR_CHAR;
d_cache_root += CACHE_ROOT;
}
// Test d_hhtp_cache_table because this method can be called before that
// instance is created and also can be called later to change the cache
// root. jhrg 05.14.08
if (d_http_cache_table)
d_http_cache_table->set_cache_root(d_cache_root);
}
void
HTTPCache::set_cache_enabled(bool mode)
{
lock_cache_interface();
d_cache_enabled = mode;
unlock_cache_interface();
}
bool
HTTPCache::is_cache_enabled() const
{
DBG2(cerr << "In HTTPCache::is_cache_enabled: (" << d_cache_enabled << ")"
<< endl);
return d_cache_enabled;
}
void
HTTPCache::set_cache_disconnected(CacheDisconnectedMode mode)
{
lock_cache_interface();
d_cache_disconnected = mode;
unlock_cache_interface();
}
CacheDisconnectedMode
HTTPCache::get_cache_disconnected() const
{
return d_cache_disconnected;
}
void
HTTPCache::set_expire_ignored(bool mode)
{
lock_cache_interface();
d_expire_ignored = mode;
unlock_cache_interface();
}
/* Is the cache ignoring Expires headers returned with responses that have
been cached? */
bool
HTTPCache::is_expire_ignored() const
{
return d_expire_ignored;
}
void
HTTPCache::set_max_size(unsigned long size)
{
lock_cache_interface();
try {
unsigned long new_size = size < MIN_CACHE_TOTAL_SIZE ?
MIN_CACHE_TOTAL_SIZE * MEGA : size * MEGA;
unsigned long old_size = d_total_size;
d_total_size = new_size;
d_folder_size = d_total_size / CACHE_FOLDER_PCT;
d_gc_buffer = d_total_size / CACHE_GC_PCT;
if (new_size < old_size && startGC()) {
perform_garbage_collection();
d_http_cache_table->cache_index_write();
}
}
catch (...) {
unlock_cache_interface();
DBGN(cerr << "Unlocking interface." << endl);
throw;
}
DBG2(cerr << "Cache....... Total cache size: " << d_total_size
<< " with " << d_folder_size
<< " bytes for meta information and folders and at least "
<< d_gc_buffer << " bytes free after every gc" << endl);
unlock_cache_interface();
}
unsigned long
HTTPCache::get_max_size() const
{
return d_total_size / MEGA;
}
void
HTTPCache::set_max_entry_size(unsigned long size)
{
lock_cache_interface();
try {
unsigned long new_size = size * MEGA;
if (new_size > 0 && new_size < d_total_size - d_folder_size) {
unsigned long old_size = d_max_entry_size;
d_max_entry_size = new_size;
if (new_size < old_size && startGC()) {
perform_garbage_collection();
d_http_cache_table->cache_index_write();
}
}
}
catch (...) {
unlock_cache_interface();
throw;
}
DBG2(cerr << "Cache...... Max entry cache size is "
<< d_max_entry_size << endl);
unlock_cache_interface();
}
unsigned long
HTTPCache::get_max_entry_size() const
{
return d_max_entry_size / MEGA;
}
void
HTTPCache::set_default_expiration(const int exp_time)
{
lock_cache_interface();
d_default_expiration = exp_time;
unlock_cache_interface();
}
int
HTTPCache::get_default_expiration() const
{
return d_default_expiration;
}
void
HTTPCache::set_always_validate(bool validate)
{
d_always_validate = validate;
}
bool
HTTPCache::get_always_validate() const
{
return d_always_validate;
}
void
HTTPCache::set_cache_control(const vector<string> &cc)
{
lock_cache_interface();
try {
d_cache_control = cc;
vector<string>::const_iterator i;
for (i = cc.begin(); i != cc.end(); ++i) {
string header = (*i).substr(0, (*i).find(':'));
string value = (*i).substr((*i).find(": ") + 2);
if (header != "Cache-Control") {
throw InternalErr(__FILE__, __LINE__, "Expected cache control header not found.");
}
else {
if (value == "no-cache" || value == "no-store")
d_cache_enabled = false;
else if (value.find("max-age") != string::npos) {
string max_age = value.substr(value.find("=") + 1);
d_max_age = parse_time(max_age.c_str());
}
else if (value == "max-stale")
d_max_stale = 0; // indicates will take anything;
else if (value.find("max-stale") != string::npos) {
string max_stale = value.substr(value.find("=") + 1);
d_max_stale = parse_time(max_stale.c_str());
}
else if (value.find("min-fresh") != string::npos) {
string min_fresh = value.substr(value.find("=") + 1);
d_min_fresh = parse_time(min_fresh.c_str());
}
}
}
}
catch (...) {
unlock_cache_interface();
throw;
}
unlock_cache_interface();
}
vector<string>
HTTPCache::get_cache_control()
{
return d_cache_control;
}
bool
HTTPCache::is_url_in_cache(const string &url)
{
DBG(cerr << "Is this url in the cache? (" << url << ")" << endl);
HTTPCacheTable::CacheEntry *entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
bool status = entry != 0;
if (entry) {
entry->unlock_read_response();
}
return status;
}
bool
is_hop_by_hop_header(const string &header)
{
return header.find("Connection") != string::npos
|| header.find("Keep-Alive") != string::npos
|| header.find("Proxy-Authenticate") != string::npos
|| header.find("Proxy-Authorization") != string::npos
|| header.find("Transfer-Encoding") != string::npos
|| header.find("Upgrade") != string::npos;
}
void
HTTPCache::write_metadata(const string &cachename, const vector<string> &headers)
{
string fname = cachename + CACHE_META;
d_open_files.push_back(fname);
FILE *dest = fopen(fname.c_str(), "w");
if (!dest) {
throw InternalErr(__FILE__, __LINE__,
"Could not open named cache entry file.");
}
vector<string>::const_iterator i;
for (i = headers.begin(); i != headers.end(); ++i) {
if (!is_hop_by_hop_header(*i)) {
int s = fwrite((*i).c_str(), (*i).size(), 1, dest);
if (s != 1) {
fclose(dest);
throw InternalErr(__FILE__, __LINE__, "could not write header: '" + (*i) + "' " + long_to_string(s));
}
s = fwrite("\n", 1, 1, dest);
if (s != 1) {
fclose(dest);
throw InternalErr(__FILE__, __LINE__, "could not write header: " + long_to_string(s));
}
}
}
int res = fclose(dest);
if (res) {
DBG(cerr << "HTTPCache::write_metadata - Failed to close "
<< dest << endl);
}
d_open_files.pop_back();
}
void
HTTPCache::read_metadata(const string &cachename, vector<string> &headers)
{
FILE *md = fopen(string(cachename + CACHE_META).c_str(), "r");
if (!md) {
throw InternalErr(__FILE__, __LINE__,
"Could not open named cache entry meta data file.");
}
const size_t line_buf_len = 1024;
char line[line_buf_len];
while (!feof(md) && fgets(line, line_buf_len, md)) {
line[std::min(line_buf_len, strnlen(line, line_buf_len))-1] = '\0'; // erase newline
headers.push_back(string(line));
}
int res = fclose(md);
if (res) {
DBG(cerr << "HTTPCache::read_metadata - Failed to close "
<< md << endl);
}
}
int
HTTPCache::write_body(const string &cachename, const FILE *src)
{
d_open_files.push_back(cachename);
FILE *dest = fopen(cachename.c_str(), "wb");
if (!dest) {
throw InternalErr(__FILE__, __LINE__,
"Could not open named cache entry file.");
}
// Read and write in 1k blocks; an attempt at doing this efficiently.
// 09/30/02 jhrg
char line[1024];
size_t n;
int total = 0;
while ((n = fread(line, 1, 1024, const_cast<FILE *>(src))) > 0) {
total += fwrite(line, 1, n, dest);
DBG2(sleep(3));
}
if (ferror(const_cast<FILE *>(src)) || ferror(dest)) {
int res = fclose(dest);
res = res & unlink(cachename.c_str());
if (res) {
DBG(cerr << "HTTPCache::write_body - Failed to close/unlink "
<< dest << endl);
}
throw InternalErr(__FILE__, __LINE__,
"I/O error transferring data to the cache.");
}
rewind(const_cast<FILE *>(src));
int res = fclose(dest);
if (res) {
DBG(cerr << "HTTPCache::write_body - Failed to close "
<< dest << endl);
}
d_open_files.pop_back();
return total;
}
FILE *
HTTPCache::open_body(const string &cachename)
{
DBG(cerr << "cachename: " << cachename << endl);
FILE *src = fopen(cachename.c_str(), "rb"); // Read only
if (!src)
throw InternalErr(__FILE__, __LINE__, "Could not open cache file.");
return src;
}
bool
HTTPCache::cache_response(const string &url, time_t request_time,
const vector<string> &headers, const FILE *body)
{
lock_cache_interface();
DBG(cerr << "Caching url: " << url << "." << endl);
try {
// If this is not an http or https URL, don't cache.
if (url.find("http:") == string::npos &&
url.find("https:") == string::npos) {
unlock_cache_interface();
return false;
}
// This does nothing if url is not already in the cache. It's
// more efficient to do this than to first check and see if the entry
// exists. 10/10/02 jhrg
d_http_cache_table->remove_entry_from_cache_table(url);
HTTPCacheTable::CacheEntry *entry = new HTTPCacheTable::CacheEntry(url);
entry->lock_write_response();
try {
d_http_cache_table->parse_headers(entry, d_max_entry_size, headers); // etag, lm, date, age, expires, max_age.
if (entry->is_no_cache()) {
DBG(cerr << "Not cache-able; deleting HTTPCacheTable::CacheEntry: " << entry
<< "(" << url << ")" << endl);
entry->unlock_write_response();
delete entry; entry = 0;
unlock_cache_interface();
return false;
}
// corrected_initial_age, freshness_lifetime, response_time.
d_http_cache_table->calculate_time(entry, d_default_expiration, request_time);
d_http_cache_table->create_location(entry); // cachename, cache_body_fd
// move these write function to cache table
entry->set_size(write_body(entry->get_cachename(), body));
write_metadata(entry->get_cachename(), headers);
d_http_cache_table->add_entry_to_cache_table(entry);
entry->unlock_write_response();
}
catch (ResponseTooBigErr &e) {
// Oops. Bummer. Clean up and exit.
DBG(cerr << e.get_error_message() << endl);
REMOVE(entry->get_cachename().c_str());
REMOVE(string(entry->get_cachename() + CACHE_META).c_str());
DBG(cerr << "Too big; deleting HTTPCacheTable::CacheEntry: " << entry << "(" << url
<< ")" << endl);
entry->unlock_write_response();
delete entry; entry = 0;
unlock_cache_interface();
return false;
}
if (d_http_cache_table->get_new_entries() > DUMP_FREQUENCY) {
if (startGC())
perform_garbage_collection();
d_http_cache_table->cache_index_write(); // resets new_entries
}
}
catch (...) {
unlock_cache_interface();
throw;
}
unlock_cache_interface();
return true;
}
vector<string>
HTTPCache::get_conditional_request_headers(const string &url)
{
lock_cache_interface();
HTTPCacheTable::CacheEntry *entry = 0;
vector<string> headers;
DBG(cerr << "Getting conditional request headers for " << url << endl);
try {
entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
if (!entry)
throw Error(internal_error, "There is no cache entry for the URL: " + url);
if (entry->get_etag() != "")
headers.push_back(string("If-None-Match: ") + entry->get_etag());
if (entry->get_lm() > 0) {
time_t lm = entry->get_lm();
headers.push_back(string("If-Modified-Since: ")
+ date_time_str(&lm));
}
else if (entry->get_max_age() > 0) {
time_t max_age = entry->get_max_age();
headers.push_back(string("If-Modified-Since: ")
+ date_time_str(&max_age));
}
else if (entry->get_expires() > 0) {
time_t expires = entry->get_expires();
headers.push_back(string("If-Modified-Since: ")
+ date_time_str(&expires));
}
entry->unlock_read_response();
unlock_cache_interface();
}
catch (...) {
unlock_cache_interface();
if (entry) {
entry->unlock_read_response();
}
throw;
}
return headers;
}
struct HeaderLess: binary_function<const string&, const string&, bool>
{
bool operator()(const string &s1, const string &s2) const {
return s1.substr(0, s1.find(':')) < s2.substr(0, s2.find(':'));
}
};
void
HTTPCache::update_response(const string &url, time_t request_time,
const vector<string> &headers)
{
lock_cache_interface();
HTTPCacheTable::CacheEntry *entry = 0;
DBG(cerr << "Updating the response headers for: " << url << endl);
try {
entry = d_http_cache_table->get_write_locked_entry_from_cache_table(url);
if (!entry)
throw Error(internal_error, "There is no cache entry for the URL: " + url);
// Merge the new headers with the exiting HTTPCacheTable::CacheEntry object.
d_http_cache_table->parse_headers(entry, d_max_entry_size, headers);
// Update corrected_initial_age, freshness_lifetime, response_time.
d_http_cache_table->calculate_time(entry, d_default_expiration, request_time);
// Merge the new headers with those in the persistent store. How:
// Load the new headers into a set, then merge the old headers. Since
// set<> ignores duplicates, old headers with the same name as a new
// header will got into the bit bucket. Define a special compare
// functor to make sure that headers are compared using only their
// name and not their value too.
set<string, HeaderLess> merged_headers;
// Load in the new headers
copy(headers.begin(), headers.end(),
inserter(merged_headers, merged_headers.begin()));
// Get the old headers and load them in.
vector<string> old_headers;
read_metadata(entry->get_cachename(), old_headers);
copy(old_headers.begin(), old_headers.end(),
inserter(merged_headers, merged_headers.begin()));
// Read the values back out. Use reverse iterators with back_inserter
// to preserve header order. NB: vector<> does not support push_front
// so we can't use front_inserter(). 01/09/03 jhrg
vector<string> result;
copy(merged_headers.rbegin(), merged_headers.rend(),
back_inserter(result));
write_metadata(entry->get_cachename(), result);
entry->unlock_write_response();
unlock_cache_interface();
}
catch (...) {
if (entry) {
entry->unlock_read_response();
}
unlock_cache_interface();
throw;
}
}
bool
HTTPCache::is_url_valid(const string &url)
{
lock_cache_interface();
bool freshness;
HTTPCacheTable::CacheEntry *entry = 0;
DBG(cerr << "Is this URL valid? (" << url << ")" << endl);
try {
if (d_always_validate) {
unlock_cache_interface();
return false; // force re-validation.
}
entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
if (!entry)
throw Error(internal_error, "There is no cache entry for the URL: " + url);
// If we supported range requests, we'd need code here to check if
// there was only a partial response in the cache. 10/02/02 jhrg
// In case this entry is of type "must-revalidate" then we consider it
// invalid.
if (entry->get_must_revalidate()) {
entry->unlock_read_response();
unlock_cache_interface();
return false;
}
time_t resident_time = time(NULL) - entry->get_response_time();
time_t current_age = entry->get_corrected_initial_age() + resident_time;
// Check that the max-age, max-stale, and min-fresh directives
// given in the request cache control header is followed.
if (d_max_age >= 0 && current_age > d_max_age) {
DBG(cerr << "Cache....... Max-age validation" << endl);
entry->unlock_read_response();
unlock_cache_interface();
return false;
}
if (d_min_fresh >= 0
&& entry->get_freshness_lifetime() < current_age + d_min_fresh) {
DBG(cerr << "Cache....... Min-fresh validation" << endl);
entry->unlock_read_response();
unlock_cache_interface();
return false;
}
freshness = (entry->get_freshness_lifetime()
+ (d_max_stale >= 0 ? d_max_stale : 0) > current_age);
entry->unlock_read_response();
unlock_cache_interface();
}
catch (...) {
if (entry) {
entry->unlock_read_response();
}
unlock_cache_interface();
throw;
}
return freshness;
}
FILE * HTTPCache::get_cached_response(const string &url,
vector<string> &headers, string &cacheName) {
lock_cache_interface();
FILE *body = 0;
HTTPCacheTable::CacheEntry *entry = 0;
DBG(cerr << "Getting the cached response for " << url << endl);
try {
entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
if (!entry) {
unlock_cache_interface();
return 0;
}
cacheName = entry->get_cachename();
read_metadata(entry->get_cachename(), headers);
DBG(cerr << "Headers just read from cache: " << endl);
DBGN(copy(headers.begin(), headers.end(), ostream_iterator<string>(cerr, "\n")));
body = open_body(entry->get_cachename());
DBG(cerr << "Returning: " << url << " from the cache." << endl);
d_http_cache_table->bind_entry_to_data(entry, body);
}
catch (...) {
// Why make this unlock operation conditional on entry?
if (entry)
unlock_cache_interface();
if (body != 0)
fclose(body);
throw;
}
unlock_cache_interface();
return body;
}
FILE *
HTTPCache::get_cached_response(const string &url, vector<string> &headers)
{
string discard_name;
return get_cached_response(url, headers, discard_name);
}
FILE *
HTTPCache::get_cached_response(const string &url)
{
string discard_name;
vector<string> discard_headers;
return get_cached_response(url, discard_headers, discard_name);
}
void
HTTPCache::release_cached_response(FILE *body)
{
lock_cache_interface();
try {
// fclose(body); This results in a seg fault on linux jhrg 8/27/13
d_http_cache_table->uncouple_entry_from_data(body);
}
catch (...) {
unlock_cache_interface();
throw;
}
unlock_cache_interface();
}
void
HTTPCache::purge_cache()
{
lock_cache_interface();
try {
if (d_http_cache_table->is_locked_read_responses())
throw Error(internal_error, "Attempt to purge the cache with entries in use.");
d_http_cache_table->delete_all_entries();
}
catch (...) {
unlock_cache_interface();
throw;
}
unlock_cache_interface();
}
} // namespace libdap
libdap::Error
A class for error processing.
Definition Error.h:94
libdap::Error::get_error_message
std::string get_error_message() const
Definition Error.cc:243
libdap::EventHandler
Definition EventHandler.h:43
libdap::HTTPCacheInterruptHandler
Definition HTTPCacheInterruptHandler.h:58
libdap::HTTPCacheTable::create_location
void create_location(CacheEntry *entry)
Definition HTTPCacheTable.cc:478
libdap::HTTPCacheTable::calculate_time
void calculate_time(HTTPCacheTable::CacheEntry *entry, int default_expiration, time_t request_time)
Definition HTTPCacheTable.cc:734
libdap::HTTPCacheTable::parse_headers
void parse_headers(HTTPCacheTable::CacheEntry *entry, unsigned long max_entry_size, const vector< string > &headers)
Definition HTTPCacheTable.cc:777
libdap::HTTPCacheTable::get_write_locked_entry_from_cache_table
CacheEntry * get_write_locked_entry_from_cache_table(const string &url)
Definition HTTPCacheTable.cc:600
libdap::HTTPCacheTable::cache_index_write
void cache_index_write()
Definition HTTPCacheTable.cc:397
libdap::HTTPCacheTable::add_entry_to_cache_table
void add_entry_to_cache_table(CacheEntry *entry)
Definition HTTPCacheTable.cc:535
libdap::HTTPCacheTable::remove_entry_from_cache_table
void remove_entry_from_cache_table(const string &url)
Definition HTTPCacheTable.cc:674
libdap::HTTPCache
Definition HTTPCache.h:104
libdap::HTTPCache::get_cache_disconnected
CacheDisconnectedMode get_cache_disconnected() const
Definition HTTPCache.cc:676
libdap::HTTPCache::cache_response
bool cache_response(const string &url, time_t request_time, const vector< string > &headers, const FILE *body)
Definition HTTPCache.cc:1157
libdap::HTTPCache::instance
static HTTPCache * instance(const string &cache_root, bool force=false)
Definition HTTPCache.cc:129
libdap::HTTPCache::get_cache_control
vector< string > get_cache_control()
Definition HTTPCache.cc:920
libdap::HTTPCache::set_expire_ignored
void set_expire_ignored(bool mode)
Definition HTTPCache.cc:690
libdap::HTTPCache::set_default_expiration
void set_default_expiration(int exp_time)
Definition HTTPCache.cc:819
libdap::HTTPCache::get_cache_root
string get_cache_root() const
Definition HTTPCache.cc:516
libdap::HTTPCache::set_cache_disconnected
void set_cache_disconnected(CacheDisconnectedMode mode)
Definition HTTPCache.cc:664
libdap::HTTPCache::release_cached_response
void release_cached_response(FILE *response)
Definition HTTPCache.cc:1572
libdap::HTTPCache::get_conditional_request_headers
vector< string > get_conditional_request_headers(const string &url)
Definition HTTPCache.cc:1250
libdap::HTTPCache::get_max_entry_size
unsigned long get_max_entry_size() const
Definition HTTPCache.cc:803
libdap::HTTPCache::set_cache_enabled
void set_cache_enabled(bool mode)
Definition HTTPCache.cc:635
libdap::HTTPCache::get_max_size
unsigned long get_max_size() const
Definition HTTPCache.cc:758
libdap::HTTPCache::set_max_entry_size
void set_max_entry_size(unsigned long size)
Definition HTTPCache.cc:772
libdap::HTTPCache::get_always_validate
bool get_always_validate() const
Definition HTTPCache.cc:850
libdap::HTTPCache::get_default_expiration
int get_default_expiration() const
Definition HTTPCache.cc:831
libdap::HTTPCache::is_url_valid
bool is_url_valid(const string &url)
Definition HTTPCache.cc:1389
libdap::HTTPCache::set_always_validate
void set_always_validate(bool validate)
Definition HTTPCache.cc:841
libdap::HTTPCache::update_response
void update_response(const string &url, time_t request_time, const vector< string > &headers)
Definition HTTPCache.cc:1320
libdap::HTTPCache::set_max_size
void set_max_size(unsigned long size)
Definition HTTPCache.cc:724
libdap::HTTPCache::set_cache_control
void set_cache_control(const vector< string > &cc)
Definition HTTPCache.cc:872
libdap::HTTPCache::~HTTPCache
virtual ~HTTPCache()
Definition HTTPCache.cc:302
libdap::HTTPCache::get_cached_response
FILE * get_cached_response(const string &url, vector< string > &headers, string &cacheName)
Definition HTTPCache.cc:1481
libdap::HTTPCache::is_cache_enabled
bool is_cache_enabled() const
Definition HTTPCache.cc:647
libdap::HTTPCache::purge_cache
void purge_cache()
Definition HTTPCache.cc:1601
libdap::InternalErr
A class for software fault reporting.
Definition InternalErr.h:65
libdap::ResponseTooBigErr
Definition ResponseTooBigErr.h:46
libdap::SignalHandlerRegisteredErr
Definition SignalHandlerRegisteredErr.h:44
libdap::SignalHandler::register_handler
EventHandler * register_handler(int signum, EventHandler *eh, bool ignore_by_default=false)
Definition SignalHandler.cc:151
libdap::SignalHandler::instance
static SignalHandler * instance()
Definition SignalHandler.cc:131
libdap::SignalHandler::remove_handler
EventHandler * remove_handler(int signum)
Definition SignalHandler.cc:220
libdap
top level DAP object to house generic methods
Definition AlarmHandler.h:36
libdap::is_hop_by_hop_header
bool is_hop_by_hop_header(const string &header)
Definition HTTPCache.cc:954
libdap::CacheDisconnectedMode
CacheDisconnectedMode
Definition HTTPCacheDisconnectedMode.h:47
libdap::date_time_str
string date_time_str(time_t *calendar, bool local)
Definition util_mit.cc:273
libdap::parse_time
time_t parse_time(const char *str, bool expand)
Definition util_mit.cc:153
libdap::HTTPCacheTable::CacheEntry
Definition HTTPCacheTable.h:114