Loading...
Searching...
No Matches
XrdOssCsiPages Class Reference

`#include <XrdOssCsiPages.hh>`

Collaboration diagram for XrdOssCsiPages:

## Public Types

typedef std::pair< off_t, off_t > | Sizes_t

## Public Member Functions

| XrdOssCsiPages (const std::string &fn, std::unique_ptr< XrdOssCsiTagstore > ts, bool wh, bool am, bool dpe, bool dlw, const char *)
| ~XrdOssCsiPages ()
void | BasicConsistencyCheck (XrdOssDF *)
int | Close ()
int | FetchRange (XrdOssDF *, const void *, off_t, size_t, uint32_t *, uint64_t, XrdOssCsiRangeGuard &)
void | Flush ()
int | Fsync ()
bool | IsReadOnly () const
int | LockResetSizes (XrdOssDF *, off_t)
void | LockTrackinglen (XrdOssCsiRangeGuard &, off_t, off_t, bool)
int | Open (const char *path, off_t dsize, int flags, XrdOucEnv &envP)
int | StoreRange (XrdOssDF *, const void *, off_t, size_t, uint32_t *, uint64_t, XrdOssCsiRangeGuard &)
void | TrackedSizeRelease ()
int | TrackedSizesGet (Sizes_t &, bool)
int | truncate (XrdOssDF *, off_t, XrdOssCsiRangeGuard &)
int | UpdateRange (XrdOssDF *, const void *, off_t, size_t, XrdOssCsiRangeGuard &)
int | VerificationStatus ()
int | VerifyRange (XrdOssDF *, const void *, off_t, size_t, XrdOssCsiRangeGuard &)

## Static Public Member Functions

static void | pgDoCalc (const void *, off_t, size_t, uint32_t *)
static int | pgWritePrelockCheck (const void *, off_t, size_t, const uint32_t *, uint64_t)

## Protected Member Functions

ssize_t | apply_sequential_aligned_modify (const void *, off_t, size_t, const uint32_t *, bool, bool, uint32_t, uint32_t)
std::string | ByteMismatchError (size_t blen, off_t off, uint8_t user, uint8_t page)
std::string | CRCMismatchError (size_t blen, off_t pgnum, uint32_t got, uint32_t expected)
int | FetchRangeAligned (const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
int | FetchRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
int | FetchRangeUnaligned_postblock (XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, size_t, uint64_t)
int | FetchRangeUnaligned_preblock (XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, uint64_t)
int | LockMakeUnverified ()
int | LockSetTrackedSize (off_t)
int | LockTruncateSize (off_t, bool)
std::string | PageReadError (size_t blen, off_t pgnum, int ret)
int | StoreRangeAligned (const void *, off_t, size_t, const Sizes_t &, uint32_t *)
int | StoreRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, const uint32_t *)
int | StoreRangeUnaligned_postblock (XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
int | StoreRangeUnaligned_preblock (XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
std::string | TagsReadError (off_t start, size_t n, int ret)
std::string | TagsWriteError (off_t start, size_t n, int ret)
int | UpdateRangeAligned (const void *, off_t, size_t, const Sizes_t &)
int | UpdateRangeHoleUntilPage (XrdOssDF *, off_t, const Sizes_t &)
int | UpdateRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
int | VerifyRangeAligned (const void *, off_t, size_t, const Sizes_t &)
int | VerifyRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)

## Static Protected Member Functions

static ssize_t | fullread (XrdOssDF *fd, void *buff, const off_t off, const size_t sz)
static ssize_t | maxread (XrdOssDF *fd, void *buff, const off_t off, const size_t sz, size_t tg=0)

## Protected Attributes

bool | allowMissingTags_
bool | checklastpg_
bool | disablePgExtend_
const std::string | fn_
bool | hasMissingTags_
off_t | lastpgforloose_
bool | loosewrite_
const bool | loosewriteConfigured_
XrdSysMutex | rangeaddmtx_
XrdOssCsiRanges | ranges_
bool | rdonly_
const char * | tident
const std::string | tident_
std::unique_ptr< XrdOssCsiTagstore > | ts_
XrdSysCondVar | tscond_
bool | tsforupdate_
bool | writeHoles_

## Static Protected Attributes

static const size_t | stsize_ = 1024

## Detailed Description

Definition at line 46 of file XrdOssCsiPages.hh.

## Member Typedef Documentation

## Sizes_t

typedef std::pair<off_t,off_t> XrdOssCsiPages::Sizes_t

Definition at line 49 of file XrdOssCsiPages.hh.

## Constructor & Destructor Documentation

## XrdOssCsiPages()

XrdOssCsiPages::XrdOssCsiPages | ( | const std::string & | fn,
|  | std::unique_ptr< XrdOssCsiTagstore > | ts,
|  | bool | wh,
|  | bool | am,
|  | bool | dpe,
|  | bool | dlw,
|  | const char * | tid )

Definition at line 44 of file XrdOssCsiPages.cc.

:
ts_(std::move(ts)),
writeHoles_(wh),
allowMissingTags_(am),
disablePgExtend_(dpe),
hasMissingTags_(false),
rdonly_(false),
loosewriteConfigured_(!dlw),
loosewrite_(false),
tscond_(0),
tsforupdate_(false),
fn_(fn),
tident_(tid),
tident(tident_.c_str()),
lastpgforloose_(0),
checklastpg_(false)
{
// empty constructor
}
ts
static std::string ts()
timestamp output for logging messages
Definition XrdCephOss.cc:53
XrdOssCsiPages::tscond_
XrdSysCondVar tscond_
Definition XrdOssCsiPages.hh:91
XrdOssCsiPages::ts_
std::unique_ptr< XrdOssCsiTagstore > ts_
Definition XrdOssCsiPages.hh:80
XrdOssCsiPages::tident
const char * tident
Definition XrdOssCsiPages.hh:98
XrdOssCsiPages::writeHoles_
bool writeHoles_
Definition XrdOssCsiPages.hh:83
XrdOssCsiPages::lastpgforloose_
off_t lastpgforloose_
Definition XrdOssCsiPages.hh:101
XrdOssCsiPages::tident_
const std::string tident_
Definition XrdOssCsiPages.hh:97
XrdOssCsiPages::tsforupdate_
bool tsforupdate_
Definition XrdOssCsiPages.hh:92
XrdOssCsiPages::disablePgExtend_
bool disablePgExtend_
Definition XrdOssCsiPages.hh:85
XrdOssCsiPages::checklastpg_
bool checklastpg_
Definition XrdOssCsiPages.hh:102
XrdOssCsiPages::hasMissingTags_
bool hasMissingTags_
Definition XrdOssCsiPages.hh:86
XrdOssCsiPages::loosewrite_
bool loosewrite_
Definition XrdOssCsiPages.hh:89
XrdOssCsiPages::rdonly_
bool rdonly_
Definition XrdOssCsiPages.hh:87
XrdOssCsiPages::loosewriteConfigured_
const bool loosewriteConfigured_
Definition XrdOssCsiPages.hh:88
XrdOssCsiPages::allowMissingTags_
bool allowMissingTags_
Definition XrdOssCsiPages.hh:84
XrdOssCsiPages::fn_
const std::string fn_
Definition XrdOssCsiPages.hh:96

References allowMissingTags_, checklastpg_, disablePgExtend_, fn_, hasMissingTags_, lastpgforloose_, loosewrite_, loosewriteConfigured_, rdonly_, tident, tident_, ts(), ts_, tscond_, tsforupdate_, and writeHoles_.

Here is the call graph for this function:

## ~XrdOssCsiPages()

XrdOssCsiPages::~XrdOssCsiPages | ( |  | ) |

inline

Definition at line 52 of file XrdOssCsiPages.hh.

{ (void)Close(); }
XrdOssCsiPages::Close
int Close()
Definition XrdOssCsiPages.cc:88

References Close().

Here is the call graph for this function:

## Member Function Documentation

## apply_sequential_aligned_modify()

ssize_t XrdOssCsiPages::apply_sequential_aligned_modify | ( | const void * | buff,
|  | off_t | startp,
|  | size_t | nbytes,
|  | const uint32_t * | csvec,
|  | bool | preblockset,
|  | bool | lastblockset,
|  | uint32_t | cspre,
|  | uint32_t | cslast )

protected

Definition at line 288 of file XrdOssCsiPages.cc.

{
EPNAME("apply_sequential_aligned_modify");
if (lastblockset && (nbytes % XrdSys::PageSize)==0)
{
return -EINVAL;
}
if (preblockset && startp==0)
{
return -EINVAL;
}
uint32_t calcbuf[stsize_];
const size_t calcbufsz = sizeof(calcbuf)/sizeof(uint32_t);
const uint8_t *const p = (uint8_t*)buff;
// will be using calcbuf
bool useinternal = true;
if (csvec && !preblockset && !lastblockset)
{
useinternal = false;
}
bool dopre = preblockset;
const off_t sp = preblockset ? startp-1 : startp;
size_t blktowrite = ((nbytes+XrdSys::PageSize-1)/XrdSys::PageSize) + (preblockset ? 1 : 0);
size_t nblkwritten = 0;
size_t calcbytot = 0;
while(blktowrite>0)
{
size_t blkwcnt = blktowrite;
if (useinternal)
{
size_t cidx = 0;
size_t calcbycnt = nbytes - calcbytot;
if (nblkwritten == 0 && dopre)
{
calcbycnt = std::min(calcbycnt, (calcbufsz-1)*XrdSys::PageSize);
blkwcnt = (calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize;
calcbuf[cidx] = cspre;
cidx++;
blkwcnt++;
dopre = false;
}
else
{
calcbycnt = std::min(calcbycnt, calcbufsz*XrdSys::PageSize);
blkwcnt = (calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize;
}
if ((calcbycnt % XrdSys::PageSize)!=0 && lastblockset)
{
const size_t x = calcbycnt / XrdSys::PageSize;
calcbycnt = XrdSys::PageSize * x;
calcbuf[cidx + x] = cslast;
}
if (csvec)
{
memcpy(&calcbuf[cidx], &csvec[calcbytot/XrdSys::PageSize], 4*((calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize));
}
else
{
XrdOucCRC::Calc32C(&p[calcbytot], calcbycnt, &calcbuf[cidx]);
}
calcbytot += calcbycnt;
}
const ssize_t wret = ts_->WriteTags(useinternal ? calcbuf : &csvec[nblkwritten], sp+nblkwritten, blkwcnt);
if (wret<0)
{
TRACE(Warn, TagsWriteError(sp+nblkwritten, blkwcnt, wret));
return wret;
}
blktowrite -= blkwcnt;
nblkwritten += blkwcnt;
}
return nblkwritten;
}
EPNAME
#define EPNAME(x)
Definition XrdBwmTrace.hh:56
TRACE
#define TRACE(act, x)
Definition XrdTrace.hh:63
XrdOssCsiPages::TagsWriteError
std::string TagsWriteError(off_t start, size_t n, int ret)
Definition XrdOssCsiPages.hh:195
XrdOssCsiPages::stsize_
static const size_t stsize_
Definition XrdOssCsiPages.hh:204
XrdOucCRC::Calc32C
static uint32_t Calc32C(const void *data, size_t count, uint32_t prevcs=0)
Definition XrdOucCRC.cc:190
XrdSys::PageSize
static const int PageSize
Definition XrdSysPageSize.hh:36

References XrdOucCRC::Calc32C(), EPNAME, XrdSys::PageSize, stsize_, TagsWriteError(), TRACE, and ts_.

Referenced by StoreRangeAligned(), and StoreRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## BasicConsistencyCheck()

void XrdOssCsiPages::BasicConsistencyCheck | ( | XrdOssDF * | fd | ) |

Definition at line 820 of file XrdOssCsiPages.cc.

{
EPNAME("BasicConsistencyCheck");
if (!loosewrite_ || rdonly_) return;
uint8_t b[XrdSys::PageSize];
static const uint8_t bz[XrdSys::PageSize] = {0};
const off_t tagsize = ts_->GetTrackedTagSize();
const off_t datasize = ts_->GetTrackedDataSize();
off_t taglp = 0, datalp = 0;
size_t tag_len = 0, data_len = 0;
if (tagsize>0)
{
taglp = (tagsize - 1) / XrdSys::PageSize;
tag_len = tagsize % XrdSys::PageSize;
tag_len = tag_len ? tag_len : XrdSys::PageSize;
}
if (datasize>0)
{
datalp = (datasize - 1) / XrdSys::PageSize;
data_len = datasize % XrdSys::PageSize;
data_len = data_len ? data_len : XrdSys::PageSize;
}
lastpgforloose_ = taglp;
checklastpg_ = true;
if (datasize>0 && taglp > datalp)
{
ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * datalp, XrdSys::PageSize);
if (rlen<0)
{
TRACE(Warn, PageReadError(XrdSys::PageSize, datalp, rlen));
return;
}
memset(&b[rlen], 0, XrdSys::PageSize-rlen);
const uint32_t data_crc = XrdOucCRC::Calc32C(b, data_len, 0u);
const uint32_t data_crc_z = XrdOucCRC::Calc32C(b, XrdSys::PageSize, 0u);
uint32_t tagv;
ssize_t rret = ts_->ReadTags(&tagv, datalp, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(datalp, 1, rret));
return;
}
if (tagv == data_crc_z)
{
// expected
}
else if (tagv == data_crc)
{
// should set tagv to data_crc_z
TRACE(Warn, "Resetting tag for page at " << datalp*XrdSys::PageSize << " to zero-extended");
const ssize_t wret = ts_->WriteTags(&data_crc_z, datalp, 1);
if (wret < 0)
{
TRACE(Warn, TagsWriteError(datalp, 1, wret));
return;
}
}
else
{
// something else wrong
TRACE(Warn, CRCMismatchError(data_len, datalp, data_crc, tagv) << " (ignoring)");
}
}
else if (tagsize>0 && taglp < datalp)
{
// datafile has more pages than recorded in the tag file:
// the tag file should have a crc corresponding to the relevant data fragment that is tracked in the last page.
// If it has the crc for a whole page (and there no non-zero content later in the page) reset it.
// This is so that a subsequnt UpdateRangeHoleUntilPage can zero-extend the CRC and get a consistent CRC.
ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * taglp, XrdSys::PageSize);
if (rlen<0)
{
TRACE(Warn, PageReadError(XrdSys::PageSize, taglp, rlen));
return;
}
memset(&b[rlen], 0, XrdSys::PageSize-rlen);
const uint32_t tag_crc = XrdOucCRC::Calc32C(b, tag_len, 0u);
const uint32_t tag_crc_z = XrdOucCRC::Calc32C(b, XrdSys::PageSize, 0u);
const uint32_t dp_ext_is_zero = !memcmp(&b[tag_len], bz, XrdSys::PageSize-tag_len);
uint32_t tagv;
ssize_t rret = ts_->ReadTags(&tagv, taglp, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(taglp, 1, rret));
return;
}
if (tagv == tag_crc)
{
// expected
}
else if (tagv == tag_crc_z && dp_ext_is_zero)
{
// should set tagv to tag_crc
TRACE(Warn, "Resetting tag for page at " << taglp*XrdSys::PageSize << " to not zero-extended");
const ssize_t wret = ts_->WriteTags(&tag_crc, taglp, 1);
if (wret < 0)
{
TRACE(Warn, TagsWriteError(taglp, 1, wret));
return;
}
}
else
{
// something else wrong
TRACE(Warn, CRCMismatchError(tag_len, taglp, tag_crc, tagv) << " dp_ext_is_zero=" << dp_ext_is_zero << " (ignoring)");
}
}
}
XrdOssCsiPages::maxread
static ssize_t maxread(XrdOssDF *fd, void *buff, const off_t off, const size_t sz, size_t tg=0)
Definition XrdOssCsiPages.hh:132
XrdOssCsiPages::TagsReadError
std::string TagsReadError(off_t start, size_t n, int ret)
Definition XrdOssCsiPages.hh:186
XrdOssCsiPages::CRCMismatchError
std::string CRCMismatchError(size_t blen, off_t pgnum, uint32_t got, uint32_t expected)
Definition XrdOssCsiPages.hh:148
XrdOssCsiPages::PageReadError
std::string PageReadError(size_t blen, off_t pgnum, int ret)
Definition XrdOssCsiPages.hh:174

References XrdOucCRC::Calc32C(), checklastpg_, CRCMismatchError(), EPNAME, lastpgforloose_, loosewrite_, maxread(), PageReadError(), XrdSys::PageSize, rdonly_, TagsReadError(), TagsWriteError(), TRACE, and ts_.

Referenced by LockResetSizes().

Here is the call graph for this function:
Here is the caller graph for this function:

## ByteMismatchError()

std::string XrdOssCsiPages::ByteMismatchError | ( | size_t | blen,
|  | off_t | off,
|  | uint8_t | user,
|  | uint8_t | page )

inline protected

Definition at line 161 of file XrdOssCsiPages.hh.

{
char buf[256],buf2[256];
snprintf(buf, sizeof(buf),
"unexpected byte mismatch between user-buffer and page/0x%04" PRIx32 " in file ",
(uint32_t)blen);
snprintf(buf2, sizeof(buf2),
" at offset 0x%" PRIx64 ", user-byte 0x%02" PRIx8 ", page-byte 0x%02" PRIx8,
(uint64_t)off,
user, page);
return buf + fn_ + buf2;
}

References fn_.

Referenced by FetchRangeUnaligned_postblock(), and FetchRangeUnaligned_preblock().

Here is the caller graph for this function:

## Close()

int XrdOssCsiPages::Close | ( |  | ) |

Definition at line 88 of file XrdOssCsiPages.cc.

{
if (hasMissingTags_)
{
hasMissingTags_ = false;
return 0;
}
return ts_->Close();
}

References hasMissingTags_, and ts_.

Referenced by ~XrdOssCsiPages().

Here is the caller graph for this function:

## CRCMismatchError()

std::string XrdOssCsiPages::CRCMismatchError | ( | size_t | blen,
|  | off_t | pgnum,
|  | uint32_t | got,
|  | uint32_t | expected )

inline protected

Definition at line 148 of file XrdOssCsiPages.hh.

{
char buf[256],buf2[256];
snprintf(buf, sizeof(buf),
"bad crc32c/0x%04" PRIx32 " checksum in file ",
(uint32_t)blen);
snprintf(buf2, sizeof(buf2),
" at offset 0x%" PRIx64 ", got 0x%08" PRIx32 ", expected 0x%08" PRIx32,
(uint64_t)(pgnum*XrdSys::PageSize),
got, expected);
return buf + fn_ + buf2;
}

References fn_, and XrdSys::PageSize.

Referenced by BasicConsistencyCheck(), FetchRangeAligned(), FetchRangeUnaligned(), FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

Here is the caller graph for this function:

## FetchRange()

int XrdOssCsiPages::FetchRange | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | uint32_t * | csvec,
|  | uint64_t | opts,
|  | XrdOssCsiRangeGuard & | rg )

Definition at line 627 of file XrdOssCsiPages.cc.

{
EPNAME("FetchRange");
if (offset<0)
{
return -EINVAL;
}
// if the tag file is missing there is nothing to fetch or verify
// but if we should return a list of checksums calculate them from the data
if (hasMissingTags_)
{
if (csvec)
{
pgDoCalc(buff, offset, blen, csvec);
}
return 0;
}
const Sizes_t sizes = rg.getTrackinglens();
const off_t trackinglen = sizes.first;
if (offset >= trackinglen && blen == 0)
{
return 0;
}
if (blen == 0)
{
// if offset is before the tracked len we should not be requested to verify zero bytes:
// the file may have been truncated
TRACE(Warn, "Fetch request for zero bytes " << fn_ << ", file may be truncated");
return -EDOM;
}
if (offset+blen > static_cast<size_t>(trackinglen))
{
TRACE(Warn, "Fetch request for " << (offset+blen-trackinglen) << " bytes from " << fn_ << " beyond tracked length");
return -EDOM;
}
if (csvec == NULL && !(opts & XrdOssDF::Verify))
{
// if the crc values are not wanted nor checks against data, then
// there's nothing more to do here
return 0;
}
int fret;
if ((offset % XrdSys::PageSize) != 0 || (offset+blen != static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0))
{
fret = FetchRangeUnaligned(fd, buff, offset, blen, sizes, csvec, opts);
}
else
{
fret = FetchRangeAligned(buff,offset,blen,sizes,csvec,opts);
}
return fret;
}
opts
struct myOpts opts
XrdOssCsiPages::FetchRangeAligned
int FetchRangeAligned(const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
Definition XrdOssCsiPages.cc:375
XrdOssCsiPages::Sizes_t
std::pair< off_t, off_t > Sizes_t
Definition XrdOssCsiPages.hh:49
XrdOssCsiPages::FetchRangeUnaligned
int FetchRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
Definition XrdOssCsiPagesUnaligned.cc:803
XrdOssCsiPages::pgDoCalc
static void pgDoCalc(const void *, off_t, size_t, uint32_t *)
Definition XrdOssCsiPages.cc:777
XrdOssCsiRangeGuard::getTrackinglens
const std::pair< off_t, off_t > & getTrackinglens() const
Definition XrdOssCsiRanges.hh:71
XrdOssDF::Verify
static const uint64_t Verify
all: Verify checksums
Definition XrdOss.hh:223

References EPNAME, FetchRangeAligned(), FetchRangeUnaligned(), fn_, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, opts, XrdSys::PageSize, pgDoCalc(), TRACE, and XrdOssDF::Verify.

Referenced by XrdOssCsiFile::pgRead().

Here is the call graph for this function:
Here is the caller graph for this function:

## FetchRangeAligned()

int XrdOssCsiPages::FetchRangeAligned | ( | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | ,
|  | uint32_t * | csvec,
|  | uint64_t | opts )

protected

Definition at line 375 of file XrdOssCsiPages.cc.

{
EPNAME("FetchRangeAligned");
uint32_t rdvec[stsize_],vrbuf[stsize_];
const off_t p1 = offset / XrdSys::PageSize;
const off_t p2 = (offset+blen) / XrdSys::PageSize;
const size_t p2_off = (offset+blen) % XrdSys::PageSize;
const size_t nfull = p2-p1;
uint32_t *rdbuf;
size_t rdbufsz;
if (csvec == NULL)
{
// use fixed sized stack buffer
rdbuf = rdvec;
rdbufsz = sizeof(rdvec)/sizeof(uint32_t);
}
else
{
// use supplied buffer, assumed to be large enough
rdbuf = csvec;
rdbufsz = (p2_off==0) ? nfull : (nfull+1);
}
// always use stack based, fixed sized buffer for verify
const size_t vrbufsz = sizeof(vrbuf)/sizeof(uint32_t);
// pointer to data
const uint8_t *const p = (uint8_t*)buff;
// process full pages + any partial page
size_t toread = (p2_off>0) ? nfull+1 : nfull;
size_t nread = 0;
while(toread>0)
{
const size_t rcnt = std::min(toread, rdbufsz-(nread%rdbufsz));
const ssize_t rret = ts_->ReadTags(&rdbuf[nread%rdbufsz], p1+nread, rcnt);
if (rret<0)
{
TRACE(Warn, TagsReadError(p1+nread, rcnt, rret));
return rret;
}
if ((opts & XrdOssDF::Verify))
{
size_t toverif = rcnt;
size_t nverif = 0;
while(toverif>0)
{
const size_t vcnt = std::min(toverif, vrbufsz);
const size_t databytes = (nread+nverif+vcnt <= nfull) ? (vcnt*XrdSys::PageSize) : ((vcnt-1)*XrdSys::PageSize+p2_off);
XrdOucCRC::Calc32C(&p[XrdSys::PageSize*(nread+nverif)],databytes,vrbuf);
if (memcmp(vrbuf, &rdbuf[(nread+nverif)%rdbufsz], 4*vcnt))
{
size_t badpg;
for(badpg=0;badpg<vcnt;++badpg) { if (memcmp(&vrbuf[badpg],&rdbuf[(nread+nverif+badpg)%rdbufsz],4)) break; }
TRACE(Warn, CRCMismatchError( (nread+nverif+badpg<nfull) ? XrdSys::PageSize : p2_off,
(p1+nread+nverif+badpg),
vrbuf[badpg],
rdbuf[(nread+nverif+badpg)%rdbufsz] ));
return -EDOM;
}
toverif -= vcnt;
nverif += vcnt;
}
}
toread -= rcnt;
nread += rcnt;
}
return 0;
}

References XrdOucCRC::Calc32C(), CRCMismatchError(), EPNAME, opts, XrdSys::PageSize, stsize_, TagsReadError(), TRACE, ts_, and XrdOssDF::Verify.

Referenced by FetchRange(), and VerifyRangeAligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## FetchRangeUnaligned()

int XrdOssCsiPages::FetchRangeUnaligned | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes,
|  | uint32_t * | csvec,
|  | uint64_t | opts )

protected

Definition at line 803 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("FetchRangeUnaligned");
const off_t p1 = offset / XrdSys::PageSize;
const size_t p1_off = offset % XrdSys::PageSize;
const off_t p2 = (offset+blen) / XrdSys::PageSize;
const size_t p2_off = (offset+blen) % XrdSys::PageSize;
const off_t trackinglen = sizes.first;
size_t ntagstoread = (p2_off>0) ? p2-p1+1 : p2-p1;
size_t ntagsbase = p1;
uint32_t tbufint[stsize_], *tbuf=0;
size_t tbufsz = 0;
if (!csvec)
{
tbuf = tbufint;
tbufsz = sizeof(tbufint)/sizeof(uint32_t);
}
else
{
tbuf = csvec;
tbufsz = ntagstoread;
}
size_t tcnt = std::min(ntagstoread, tbufsz);
ssize_t rret = ts_->ReadTags(tbuf, ntagsbase, tcnt);
if (rret<0)
{
TRACE(Warn, TagsReadError(ntagsbase, tcnt, rret) << " (first)");
return rret;
}
ntagstoread -= tcnt;
// deal with partial first page
if ( p1_off>0 || blen < static_cast<size_t>(XrdSys::PageSize) )
{
const int ret = FetchRangeUnaligned_preblock(fd, buff, offset, blen, trackinglen, tbuf, csvec, opts);
if (ret<0)
{
return ret;
}
}
// first (inclusive) and last (exclusive) full page
const off_t fp = (p1_off != 0) ? p1+1 : p1;
const off_t lp = p2;
// verify full pages if wanted
if (fp<lp && (opts & XrdOssDF::Verify))
{
const uint8_t *const p = (uint8_t*)buff;
uint32_t calcbuf[stsize_];
const size_t cbufsz = sizeof(calcbuf)/sizeof(uint32_t);
size_t toread = lp-fp;
size_t nread = 0;
while(toread>0)
{
const size_t ccnt = std::min(toread, cbufsz);
XrdOucCRC::Calc32C(&p[(p1_off ? XrdSys::PageSize-p1_off : 0)+XrdSys::PageSize*nread],ccnt*XrdSys::PageSize,calcbuf);
size_t tovalid = ccnt;
size_t nvalid = 0;
while(tovalid>0)
{
const size_t tidx=fp+nread+nvalid - ntagsbase;
const size_t nv = std::min(tovalid, tbufsz-tidx);
if (nv == 0)
{
assert(csvec == NULL);
ntagsbase += tbufsz;
tcnt = std::min(ntagstoread, tbufsz);
rret = ts_->ReadTags(tbuf, ntagsbase, tcnt);
if (rret<0)
{
TRACE(Warn, TagsReadError(ntagsbase, tcnt, rret) << " (mid)");
return rret;
}
ntagstoread -= tcnt;
continue;
}
if (memcmp(&calcbuf[nvalid], &tbuf[tidx], 4*nv))
{
size_t badpg;
for(badpg=0;badpg<nv;badpg++) { if (memcmp(&calcbuf[nvalid+badpg], &tbuf[tidx+badpg],4)) break; }
TRACE(Warn, CRCMismatchError(XrdSys::PageSize,
(ntagsbase+tidx+badpg),
calcbuf[nvalid+badpg], tbuf[tidx+badpg]));
return -EDOM;
}
tovalid -= nv;
nvalid += nv;
}
toread -= ccnt;
nread += ccnt;
}
}
// last partial page
if (p2>p1 && p2_off > 0)
{
// make sure we have last tag;
// (should already have all of them if we're returning them in csvec)
size_t tidx = p2 - ntagsbase;
if (tidx >= tbufsz)
{
assert(csvec == NULL);
tidx = 0;
ntagsbase = p2;
rret = ts_->ReadTags(tbuf, ntagsbase, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(ntagsbase, 1, rret) << " (last)");
return rret;
}
ntagstoread = 0;
}
const int ret = FetchRangeUnaligned_postblock(fd, buff, offset, blen, trackinglen, tbuf, csvec, tidx, opts);
if (ret<0)
{
return ret;
}
}
return 0;
}
XrdOssCsiPages::FetchRangeUnaligned_preblock
int FetchRangeUnaligned_preblock(XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, uint64_t)
Definition XrdOssCsiPagesUnaligned.cc:639
XrdOssCsiPages::FetchRangeUnaligned_postblock
int FetchRangeUnaligned_postblock(XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, size_t, uint64_t)
Definition XrdOssCsiPagesUnaligned.cc:727

References XrdOucCRC::Calc32C(), CRCMismatchError(), EPNAME, FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), opts, XrdSys::PageSize, stsize_, TagsReadError(), TRACE, ts_, and XrdOssDF::Verify.

Referenced by FetchRange(), and VerifyRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## FetchRangeUnaligned_postblock()

int XrdOssCsiPages::FetchRangeUnaligned_postblock | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | off_t | trackinglen,
|  | uint32_t * | tbuf,
|  | uint32_t * | csvec,
|  | size_t | tidx,
|  | uint64_t | opts )

protected

Definition at line 727 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("FetchRangeUnaligned_postblock");
const off_t p2 = (offset+blen) / XrdSys::PageSize;
const size_t p2_off = (offset+blen) % XrdSys::PageSize;
// length of data in last (p2) page
const size_t bavail = std::min(trackinglen - (XrdSys::PageSize*p2), (off_t)XrdSys::PageSize);
// how much of that data is not being returned
const size_t bremain = (p2_off < bavail) ? bavail-p2_off : 0;
uint8_t b[XrdSys::PageSize];
const uint8_t *ub = &((uint8_t*)buff)[blen-p2_off];
if (bremain>0)
{
const ssize_t rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize*p2, bavail);
if (rret<0)
{
TRACE(Warn, PageReadError(bavail, p2, rret));
return rret;
}
// if we're verifying make sure overlapping part of data just read matches user's buffer
if ((opts & XrdOssDF::Verify))
{
const uint8_t *const p = (uint8_t*)buff;
if (memcmp(&p[blen-p2_off], b, p2_off))
{
size_t badoff;
for(badoff=0;badoff<p2_off;badoff++) { if (p[blen-p2_off+badoff] != b[badoff]) break; }
badoff = (badoff < p2_off) ? badoff : 0; // may be possible with concurrent modification
TRACE(Warn, ByteMismatchError(bavail, XrdSys::PageSize*p2+badoff, p[blen-p2_off+badoff], b[badoff]));
return -EDOM;
}
}
ub = b;
}
if ((opts & XrdOssDF::Verify))
{
const uint32_t crc32calc = XrdOucCRC::Calc32C(ub, bavail, 0U);
if (tbuf[tidx] != crc32calc)
{
TRACE(Warn, CRCMismatchError(bavail, p2, crc32calc, tbuf[tidx]));
return -EDOM;
}
}
// if we're returning csvec and user only request part of page
// adjust the crc
if (csvec && bremain>0)
{
if ((opts & XrdOssDF::Verify))
{
// verified; calculate crc based on common part of page.
csvec[tidx] = XrdOucCRC::Calc32C(b, p2_off, 0u);
}
else
{
// recalculate crc based on recorded checksum and adjusting for part of data not returned.
// If either the returned data or the data not included in the user's request are
// corrupt the returned checksum and returned data will (probably) mismatch.
const uint32_t crc32c = XrdOucCRC::Calc32C(&b[p2_off], bremain, 0u);
csvec[tidx] = CrcUtils.crc32c_split1(csvec[tidx], crc32c, bremain);
}
}
return 0;
}
CrcUtils
static XrdOssCsiCrcUtils CrcUtils
Definition XrdOssCsiPagesUnaligned.cc:42
crc32c
uint32_t crc32c(uint32_t crc, void const *buf, size_t len)
Definition XrdOucCRC32C.cc:277
XrdOssCsiCrcUtils::crc32c_split1
static uint32_t crc32c_split1(uint32_t crctot, uint32_t crc2, size_t len2)
Definition XrdOssCsiCrcUtils.hh:69
XrdOssCsiPages::ByteMismatchError
std::string ByteMismatchError(size_t blen, off_t off, uint8_t user, uint8_t page)
Definition XrdOssCsiPages.hh:161
XrdOssCsiPages::fullread
static ssize_t fullread(XrdOssDF *fd, void *buff, const off_t off, const size_t sz)
Definition XrdOssCsiPages.hh:123

References ByteMismatchError(), XrdOucCRC::Calc32C(), crc32c(), CRCMismatchError(), CrcUtils, EPNAME, fullread(), opts, PageReadError(), XrdSys::PageSize, TRACE, and XrdOssDF::Verify.

Referenced by FetchRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## FetchRangeUnaligned_preblock()

int XrdOssCsiPages::FetchRangeUnaligned_preblock | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | off_t | trackinglen,
|  | uint32_t * | tbuf,
|  | uint32_t * | csvec,
|  | uint64_t | opts )

protected

Definition at line 639 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("FetchRangeUnaligned_preblock");
const off_t p1 = offset / XrdSys::PageSize;
const size_t p1_off = offset % XrdSys::PageSize;
// bavail is length of data in this page
const size_t bavail = std::min(trackinglen - (XrdSys::PageSize*p1), (off_t)XrdSys::PageSize);
// bcommon is length of data in this page that user wants
const size_t bcommon = std::min(bavail - p1_off, blen);
uint8_t b[XrdSys::PageSize];
const uint8_t *ub = (uint8_t*)buff;
if (bavail>bcommon)
{
// will need more data to either verify or return crc of the user's data
// (in case of no verify and no csvec FetchRange() returns early)
const ssize_t rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize*p1, bavail);
if (rret<0)
{
TRACE(Warn, PageReadError(bavail, p1, rret));
return rret;
}
// if we're going to verify, make sure we just read the same overlapping data as that in the user's buffer
if ((opts & XrdOssDF::Verify))
{
if (memcmp(buff, &b[p1_off], bcommon))
{
size_t badoff;
for(badoff=0;badoff<bcommon;badoff++) { if (((uint8_t*)buff)[badoff] != b[p1_off+badoff]) break; }
badoff = (badoff < bcommon) ? badoff : 0; // may be possible with concurrent modification
TRACE(Warn, ByteMismatchError(bavail, XrdSys::PageSize*p1+p1_off+badoff, ((uint8_t*)buff)[badoff], b[p1_off+badoff]));
return -EDOM;
}
}
ub = b;
}
// verify; based on whole block, or user's buffer (if it contains the whole block)
if ((opts & XrdOssDF::Verify))
{
const uint32_t crc32calc = XrdOucCRC::Calc32C(ub, bavail, 0U);
if (tbuf[0] != crc32calc)
{
TRACE(Warn, CRCMismatchError(bavail, p1, crc32calc, tbuf[0]));
return -EDOM;
}
}
// if we're returning csvec values and this first block
// needs adjustment because user requested a subset..
if (bavail>bcommon && csvec)
{
// make sure csvec[0] corresponds to only the data the user wanted, not whole page.
// if we have already verified the page + common part matches user's, take checksum of common.
// (Use local copy of page, perhaps less chance of accidental concurrent modification than buffer)
// Otherwise base on saved checksum.
if ((opts & XrdOssDF::Verify))
{
csvec[0] = XrdOucCRC::Calc32C(&b[p1_off], bcommon, 0u);
}
else
{
// calculate expected user checksum based on block's recorded checksum, adjusting
// for data not included in user's request. If either the returned data or the
// data not included in the user's request are corrupt the returned checksum and
// returned data will (probably) mismatch.
// remove block data before p1_off from checksum
uint32_t crc32c = XrdOucCRC::Calc32C(b, p1_off, 0u);
csvec[0] = CrcUtils.crc32c_split2(csvec[0], crc32c, bavail-p1_off);
// remove block data after p1_off+bcommon upto bavail
crc32c = XrdOucCRC::Calc32C(&b[p1_off+bcommon], bavail-p1_off-bcommon, 0u);
csvec[0] = CrcUtils.crc32c_split1(csvec[0], crc32c, bavail-p1_off-bcommon);
}
}
return 0;
}
XrdOssCsiCrcUtils::crc32c_split2
static uint32_t crc32c_split2(uint32_t crctot, uint32_t crc1, size_t len2)
Definition XrdOssCsiCrcUtils.hh:90

References ByteMismatchError(), XrdOucCRC::Calc32C(), crc32c(), CRCMismatchError(), CrcUtils, EPNAME, fullread(), opts, PageReadError(), XrdSys::PageSize, TRACE, and XrdOssDF::Verify.

Referenced by FetchRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## Flush()

void XrdOssCsiPages::Flush | ( |  | ) |

Definition at line 98 of file XrdOssCsiPages.cc.

{
if (!hasMissingTags_) ts_->Flush();
}

References hasMissingTags_, and ts_.

Referenced by XrdOssCsiFile::Flush().

Here is the caller graph for this function:

## Fsync()

int XrdOssCsiPages::Fsync | ( |  | ) |

Definition at line 103 of file XrdOssCsiPages.cc.

{
if (hasMissingTags_) return 0;
return ts_->Fsync();
}

References hasMissingTags_, and ts_.

Referenced by XrdOssCsiFile::Fsync().

Here is the caller graph for this function:

## fullread()

ssize_t XrdOssCsiPages::fullread | ( | XrdOssDF * | fd,
|  | void * | buff,
|  | const off_t | off,
|  | const size_t | sz )

inline static protected

Definition at line 123 of file XrdOssCsiPages.hh.

{
ssize_t rret = maxread(fd, buff, off, sz);
if (rret<0) return rret;
if (static_cast<size_t>(rret) != sz) return -EDOM;
return rret;
}

References maxread().

Referenced by FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

Here is the call graph for this function:
Here is the caller graph for this function:

## IsReadOnly()

bool XrdOssCsiPages::IsReadOnly | ( |  | ) | const

inline

Definition at line 68 of file XrdOssCsiPages.hh.

{ return rdonly_; }

References rdonly_.

## LockMakeUnverified()

int XrdOssCsiPages::LockMakeUnverified | ( |  | ) |

protected

Definition at line 152 of file XrdOssCsiPages.cc.

{
XrdSysCondVarHelper lck(&tscond_);
return ts_->SetUnverified();
}

References ts_, and tscond_.

Referenced by StoreRange(), and UpdateRange().

Here is the caller graph for this function:

## LockResetSizes()

int XrdOssCsiPages::LockResetSizes | ( | XrdOssDF * | fd,
|  | off_t | sz )

Definition at line 134 of file XrdOssCsiPages.cc.

{
// nothing to do is no tag file
if (hasMissingTags_) return 0;
XrdSysCondVarHelper lck(&tscond_);
const int ret = ts_->ResetSizes(sz);
loosewrite_ = loosewriteConfigured_;
BasicConsistencyCheck(fd);
return ret;
}
XrdOssCsiPages::BasicConsistencyCheck
void BasicConsistencyCheck(XrdOssDF *)
Definition XrdOssCsiPages.cc:820

References BasicConsistencyCheck(), hasMissingTags_, loosewrite_, loosewriteConfigured_, ts_, and tscond_.

Here is the call graph for this function:

## LockSetTrackedSize()

int XrdOssCsiPages::LockSetTrackedSize | ( | off_t | sz | ) |

protected

Definition at line 128 of file XrdOssCsiPages.cc.

{
XrdSysCondVarHelper lck(&tscond_);
return ts_->SetTrackedSize(sz);
}

References ts_, and tscond_.

Referenced by StoreRange(), and UpdateRange().

Here is the caller graph for this function:

## LockTrackinglen()

void XrdOssCsiPages::LockTrackinglen | ( | XrdOssCsiRangeGuard & | rg,
|  | off_t | offset,
|  | off_t | offend,
|  | bool | rdonly )

Definition at line 499 of file XrdOssCsiPages.cc.

{
// no need to lock if we don't have a tag file
if (hasMissingTags_) return;
// in case of empty range the tracking len is not copied
if (offset == offend) return;
{
XrdSysMutexHelper lck(rangeaddmtx_);
Sizes_t sizes;
(void)TrackedSizesGet(sizes, !rdonly);
// tag tracking data filesize, as recorded in the tagfile and for which the tagfile
// should be approprately sized, is sizes.first: usually the same as the in
// memory "actual" data filesize (sizes.second), but may differ after crashes or write failure.
const off_t trackinglen = sizes.first;
const off_t p1 = (offset>trackinglen ? trackinglen : offset) / XrdSys::PageSize;
bool unlock = false;
if (!rdonly && offend <= trackinglen)
{
unlock = true;
}
off_t p2 = offend / XrdSys::PageSize;
const size_t p2_off = offend % XrdSys::PageSize;
// range is exclusive
if (p2_off ==0) p2--;
ranges_.AddRange(p1, p2, rg, rdonly);
if (unlock)
{
TrackedSizeRelease();
}
rg.SetTrackingInfo(this, sizes, (!rdonly && !unlock));
}
rg.Wait();
}
XrdOssCsiPages::ranges_
XrdOssCsiRanges ranges_
Definition XrdOssCsiPages.hh:82
XrdOssCsiPages::TrackedSizeRelease
void TrackedSizeRelease()
Definition XrdOssCsiPages.cc:158
XrdOssCsiPages::rangeaddmtx_
XrdSysMutex rangeaddmtx_
Definition XrdOssCsiPages.hh:81
XrdOssCsiPages::TrackedSizesGet
int TrackedSizesGet(Sizes_t &, bool)
Definition XrdOssCsiPages.cc:109
XrdOssCsiRangeGuard::SetTrackingInfo
void SetTrackingInfo(XrdOssCsiPages *p, const std::pair< off_t, off_t > &tsizes, bool locked)
Definition XrdOssCsiRanges.hh:76
XrdOssCsiRangeGuard::Wait
void Wait()
Definition XrdOssCsiRanges.cc:52

References hasMissingTags_, XrdSys::PageSize, rangeaddmtx_, ranges_, XrdOssCsiRangeGuard::SetTrackingInfo(), TrackedSizeRelease(), TrackedSizesGet(), and XrdOssCsiRangeGuard::Wait().

Referenced by XrdOssCsiFile::Ftruncate(), XrdOssCsiFile::pgRead(), XrdOssCsiFile::pgWrite(), XrdOssCsiFile::Read(), XrdOssCsiFile::ReadRaw(), XrdOssCsiFile::ReadV(), XrdOssCsiFile::Write(), and XrdOssCsiFile::WriteV().

Here is the call graph for this function:
Here is the caller graph for this function:

## LockTruncateSize()

int XrdOssCsiPages::LockTruncateSize | ( | off_t | sz,
|  | bool | datatoo )

protected

Definition at line 146 of file XrdOssCsiPages.cc.

{
XrdSysCondVarHelper lck(&tscond_);
return ts_->Truncate(sz,datatoo);
}

References ts_, and tscond_.

Referenced by truncate().

Here is the caller graph for this function:

## maxread()

ssize_t XrdOssCsiPages::maxread | ( | XrdOssDF * | fd,
|  | void * | buff,
|  | const off_t | off,
|  | const size_t | sz,
|  | size_t | tg = 0 )

inline static protected

Definition at line 132 of file XrdOssCsiPages.hh.

{
size_t toread = sz, nread = 0;
uint8_t *p = (uint8_t*)buff;
tg = tg ? tg : sz;
while(toread>0 && nread<tg)
{
const ssize_t rret = fd->Read(&p[nread], off+nread, toread);
if (rret<0) return rret;
if (rret==0) break;
toread -= rret;
nread += rret;
}
return nread;
}
XrdOssDF::Read
virtual ssize_t Read(off_t offset, size_t size)
Definition XrdOss.hh:281

References XrdOssDF::Read().

Referenced by BasicConsistencyCheck(), fullread(), and StoreRangeUnaligned_preblock().

Here is the call graph for this function:
Here is the caller graph for this function:

## Open()

int XrdOssCsiPages::Open | ( | const char * | path,
|  | off_t | dsize,
|  | int | flags,
|  | XrdOucEnv & | envP )

Definition at line 64 of file XrdOssCsiPages.cc.

{
EPNAME("Pages::Open");
hasMissingTags_ = false;
rdonly_ = false;
int ret = ts_->Open(path, dsize, flags, envP);
if (ret == -ENOENT)
{
// no existing tag
if (allowMissingTags_)
{
TRACE(Info, "Opening with missing tagfile: " << fn_);
hasMissingTags_ = true;
return 0;
}
TRACE(Warn, "Could not open tagfile for " << fn_ << " error " << ret);
return -EDOM;
}
if (ret<0) return ret;
if ((flags & O_ACCMODE) == O_RDONLY) rdonly_ = true;
loosewrite_ = (dsize==0 && ts_->GetTrackedTagSize()==0) ? false : loosewriteConfigured_;
return 0;
}

References allowMissingTags_, EPNAME, fn_, hasMissingTags_, loosewrite_, loosewriteConfigured_, rdonly_, TRACE, and ts_.

## PageReadError()

std::string XrdOssCsiPages::PageReadError | ( | size_t | blen,
|  | off_t | pgnum,
|  | int | ret )

inline protected

Definition at line 174 of file XrdOssCsiPages.hh.

{
char buf[256],buf2[256];
snprintf(buf, sizeof(buf),
"error %d while reading page/0x%04" PRIx32 " in file ",
ret, (uint32_t)blen);
snprintf(buf2, sizeof(buf2),
" at offset 0x%" PRIx64,
(uint64_t)(pgnum*XrdSys::PageSize));
return buf + fn_ + buf2;
}

References fn_, and XrdSys::PageSize.

Referenced by BasicConsistencyCheck(), FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

Here is the caller graph for this function:

## pgDoCalc()

void XrdOssCsiPages::pgDoCalc | ( | const void * | buffer,
|  | off_t | offset,
|  | size_t | wrlen,
|  | uint32_t * | csvec )

static

Definition at line 777 of file XrdOssCsiPages.cc.

{
const size_t p_off = offset % XrdSys::PageSize;
const size_t p_alen = (p_off > 0) ? (XrdSys::PageSize - p_off) : wrlen;
if (p_alen < wrlen)
{
XrdOucCRC::Calc32C((uint8_t *)buffer+p_alen, wrlen-p_alen, &csvec[1]);
}
XrdOucCRC::Calc32C((void*)buffer, std::min(p_alen, wrlen), csvec);
}

References XrdOucCRC::Calc32C(), and XrdSys::PageSize.

Referenced by FetchRange(), and StoreRange().

Here is the call graph for this function:
Here is the caller graph for this function:

## pgWritePrelockCheck()

int XrdOssCsiPages::pgWritePrelockCheck | ( | const void * | buffer,
|  | off_t | offset,
|  | size_t | wrlen,
|  | const uint32_t * | csvec,
|  | uint64_t | opts )

static

Definition at line 788 of file XrdOssCsiPages.cc.

{
// do verify before taking locks to allow for faster fail
if (csvec && (opts & XrdOssDF::Verify))
{
uint32_t valcs;
const size_t p_off = offset % XrdSys::PageSize;
const size_t p_alen = (p_off > 0) ? (XrdSys::PageSize - p_off) : wrlen;
if (p_alen < wrlen)
{
if (XrdOucCRC::Ver32C((uint8_t *)buffer+p_alen, wrlen-p_alen, &csvec[1], valcs)>=0)
{
return -EDOM;
}
}
if (XrdOucCRC::Ver32C((void*)buffer, std::min(p_alen, wrlen), csvec, valcs)>=0)
{
return -EDOM;
}
}
return 0;
}
XrdOucCRC::Ver32C
static bool Ver32C(const void *data, size_t count, const uint32_t csval, uint32_t *csbad=0)
Definition XrdOucCRC.cc:222

References opts, XrdSys::PageSize, XrdOucCRC::Ver32C(), and XrdOssDF::Verify.

Referenced by XrdOssCsiFile::pgWrite(), and XrdOssCsiFile::pgWrite().

Here is the call graph for this function:
Here is the caller graph for this function:

## StoreRange()

int XrdOssCsiPages::StoreRange | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | uint32_t * | csvec,
|  | uint64_t | opts,
|  | XrdOssCsiRangeGuard & | rg )

Definition at line 691 of file XrdOssCsiPages.cc.

{
if (offset<0)
{
return -EINVAL;
}
if (blen == 0)
{
return 0;
}
// if the tag file is missing there is nothing to store
// but do calculate checksums to return, if requested to do so
if (hasMissingTags_)
{
if (csvec && (opts & XrdOssDF::doCalc))
{
pgDoCalc(buff, offset, blen, csvec);
}
return 0;
}
const Sizes_t sizes = rg.getTrackinglens();
const off_t trackinglen = sizes.first;
// in the original specification of pgWrite there was the idea of a logical-eof, set by
// the a pgWrite with non-page aligned length: We support an option to approximate that
// by disallowing pgWrite past the current (non page aligned) eof.
if (disablePgExtend_ && (trackinglen % XrdSys::PageSize) !=0 && offset+blen > static_cast<size_t>(trackinglen))
{
return -ESPIPE;
}
// if doCalc is set and we have a csvec buffer fill it with calculated values
if (csvec && (opts & XrdOssDF::doCalc))
{
pgDoCalc(buff, offset, blen, csvec);
}
// if no vector of crc have been given and not specifically requested to calculate,
// then mark this file as having unverified checksums
if (!csvec && !(opts & XrdOssDF::doCalc))
{
LockMakeUnverified();
}
if (offset+blen > static_cast<size_t>(trackinglen))
{
LockSetTrackedSize(offset+blen);
rg.unlockTrackinglen();
}
int ret;
if ((offset % XrdSys::PageSize) != 0 ||
(offset+blen < static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0) ||
((trackinglen % XrdSys::PageSize) !=0 && offset > trackinglen))
{
ret = StoreRangeUnaligned(fd,buff,offset,blen,sizes,csvec);
}
else
{
ret = StoreRangeAligned(buff,offset,blen,sizes,csvec);
}
return ret;
}
XrdOssCsiPages::StoreRangeUnaligned
int StoreRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, const uint32_t *)
Definition XrdOssCsiPagesUnaligned.cc:532
XrdOssCsiPages::StoreRangeAligned
int StoreRangeAligned(const void *, off_t, size_t, const Sizes_t &, uint32_t *)
Definition XrdOssCsiPages.cc:453
XrdOssCsiPages::LockSetTrackedSize
int LockSetTrackedSize(off_t)
Definition XrdOssCsiPages.cc:128
XrdOssCsiPages::LockMakeUnverified
int LockMakeUnverified()
Definition XrdOssCsiPages.cc:152
XrdOssCsiRangeGuard::unlockTrackinglen
void unlockTrackinglen()
Definition XrdOssCsiRanges.cc:58
XrdOssDF::doCalc
static const uint64_t doCalc
pgw: Calculate checksums
Definition XrdOss.hh:225

References disablePgExtend_, XrdOssDF::doCalc, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockMakeUnverified(), LockSetTrackedSize(), opts, XrdSys::PageSize, pgDoCalc(), StoreRangeAligned(), StoreRangeUnaligned(), and XrdOssCsiRangeGuard::unlockTrackinglen().

Referenced by XrdOssCsiFile::pgWrite().

Here is the call graph for this function:
Here is the caller graph for this function:

## StoreRangeAligned()

int XrdOssCsiPages::StoreRangeAligned | ( | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes,
|  | uint32_t * | csvec )

protected

Definition at line 453 of file XrdOssCsiPages.cc.

{
EPNAME("StoreRangeAligned");
// if csvec given store those values
// if no csvec then calculate against data and store
const off_t p1 = offset / XrdSys::PageSize;
const off_t trackinglen = sizes.first;
if (offset > trackinglen)
{
const int ret = UpdateRangeHoleUntilPage(NULL, p1, sizes);
if (ret<0)
{
TRACE(Warn, "Error updating tags for holes, error=" << ret);
return ret;
}
}
const ssize_t aret = apply_sequential_aligned_modify(buff, p1, blen, csvec, false, false, 0U, 0U);
if (aret<0)
{
TRACE(Warn, "Error updating tags, error=" << aret);
return aret;
}
return 0;
}
XrdOssCsiPages::apply_sequential_aligned_modify
ssize_t apply_sequential_aligned_modify(const void *, off_t, size_t, const uint32_t *, bool, bool, uint32_t, uint32_t)
Definition XrdOssCsiPages.cc:288
XrdOssCsiPages::UpdateRangeHoleUntilPage
int UpdateRangeHoleUntilPage(XrdOssDF *, off_t, const Sizes_t &)
Definition XrdOssCsiPagesUnaligned.cc:52

References apply_sequential_aligned_modify(), EPNAME, XrdSys::PageSize, TRACE, and UpdateRangeHoleUntilPage().

Referenced by StoreRange(), and UpdateRangeAligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## StoreRangeUnaligned()

int XrdOssCsiPages::StoreRangeUnaligned | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes,
|  | const uint32_t * | csvec )

protected

Definition at line 532 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("StoreRangeUnaligned");
const off_t p1 = offset / XrdSys::PageSize;
const off_t trackinglen = sizes.first;
if (offset > trackinglen)
{
const int ret = UpdateRangeHoleUntilPage(fd, p1, sizes);
if (ret<0)
{
TRACE(Warn, "Error updating tags for holes, error=" << ret);
return ret;
}
}
const size_t p1_off = offset % XrdSys::PageSize;
const size_t p2_off = (offset+blen) % XrdSys::PageSize;
bool hasprepage = false;
uint32_t prepageval;
// deal with partial first page
if ( p1_off>0 || blen < static_cast<size_t>(XrdSys::PageSize) )
{
const size_t bavail = (XrdSys::PageSize-p1_off > blen) ? blen : (XrdSys::PageSize-p1_off);
const int ret = StoreRangeUnaligned_preblock(fd, buff, bavail, offset, trackinglen, csvec, prepageval);
if (ret<0)
{
return ret;
}
hasprepage = true;
}
// next page (if any)
const off_t np = hasprepage ? p1+1 : p1;
// next page starts at buffer offset
const size_t npoff = hasprepage ? (XrdSys::PageSize - p1_off) : 0;
// anything in next page?
if (blen <= npoff)
{
// only need to write the first, partial page
if (hasprepage)
{
const ssize_t wret = ts_->WriteTags(&prepageval, p1, 1);
if (wret<0)
{
TRACE(Warn, TagsWriteError(p1, 1, wret));
return wret;
}
}
return 0;
}
const uint8_t *const p = (uint8_t*)buff;
const uint32_t *csp = csvec;
if (csp && hasprepage) csp++;
// see if there will be no old data to account for in the last page
if (p2_off == 0 || (offset + blen >= static_cast<size_t>(trackinglen)))
{
// write any precomputed prepage, then write full pages and last partial page (computing or using supplied csvec)
const ssize_t aret = apply_sequential_aligned_modify(&p[npoff], np, blen-npoff, csp, hasprepage, false, prepageval, 0U);
if (aret<0)
{
TRACE(Warn, "Error updating tags, error=" << aret);
return aret;
}
return 0;
}
// last page contains existing data that has to be read to modify it
uint32_t lastpageval;
const int ret = StoreRangeUnaligned_postblock(fd, &p[npoff], blen-npoff, offset+npoff, trackinglen, csp, lastpageval);
if (ret<0)
{
return ret;
}
// write any precomputed prepage, then write full pages (computing or using supplied csvec) and finally write precomputed last page
const ssize_t aret = apply_sequential_aligned_modify(&p[npoff], np, blen-npoff, csp, hasprepage, true, prepageval, lastpageval);
if (aret<0)
{
TRACE(Warn, "Error updating tags, error=" << aret);
return aret;
}
return 0;
}
XrdOssCsiPages::StoreRangeUnaligned_preblock
int StoreRangeUnaligned_preblock(XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
Definition XrdOssCsiPagesUnaligned.cc:148
XrdOssCsiPages::StoreRangeUnaligned_postblock
int StoreRangeUnaligned_postblock(XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
Definition XrdOssCsiPagesUnaligned.cc:431

References apply_sequential_aligned_modify(), EPNAME, XrdSys::PageSize, StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), TagsWriteError(), TRACE, ts_, and UpdateRangeHoleUntilPage().

Referenced by StoreRange(), and UpdateRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## StoreRangeUnaligned_postblock()

int XrdOssCsiPages::StoreRangeUnaligned_postblock | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | size_t | blen,
|  | off_t | offset,
|  | off_t | trackinglen,
|  | const uint32_t * | csvec,
|  | uint32_t & | lastpageval )

protected

Definition at line 431 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("StoreRangeUnaligned_postblock");
const uint8_t *const p = (uint8_t*)buff;
const off_t p2 = (offset+blen) / XrdSys::PageSize;
const size_t p2_off = (offset+blen) % XrdSys::PageSize;
const off_t tracked_page = trackinglen / XrdSys::PageSize;
const size_t tracked_off = trackinglen % XrdSys::PageSize;
// we should not be called in this case
assert(p2_off != 0);
// how much existing data this last (p2) page
const size_t bavail = (p2==tracked_page) ? tracked_off : XrdSys::PageSize;
// how much of that data will not be overwritten
const size_t bremain = (p2_off < bavail) ? bavail-p2_off : 0;
// we should not be called if it is a complete overwrite
assert(bremain>0);
// need to use remaining data to calculate the crc of the new p2 page.
// read and verify it now.
uint32_t crc32v;
ssize_t rret = ts_->ReadTags(&crc32v, p2, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(p2, 1, rret));
return rret;
}
uint8_t b[XrdSys::PageSize];
rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize * p2, bavail);
if (rret<0)
{
TRACE(Warn, PageReadError(bavail, p2, rret));
return rret;
}
// calculate crc of new data with remaining data at the end:
uint32_t crc32c = 0;
if (csvec)
{
crc32c = csvec[(blen-1)/XrdSys::PageSize];
}
else
{
crc32c = XrdOucCRC::Calc32C(&p[blen-p2_off], p2_off, 0U);
}
const uint32_t cl = XrdOucCRC::Calc32C(&b[p2_off], bremain, 0U);
// crc of page with new data
crc32c = CrcUtils.crc32c_combine(crc32c, cl, bremain);
// crc of current page (before write)
const uint32_t crc32prev = XrdOucCRC::Calc32C(b, bavail, 0U);
// check(s) to see if remaining data was valid
// usual check; unmodified block is consistent with stored crc
// for loose write we allow case were the new crc has already been stored in the tagfile
// this may be an implicit verification (e.g. pgWrite may return EDOM without Verify requested)
// however, it's not clear if there is a meaningful way to crc a mismatching page during a partial overwrite
if (crc32v != crc32prev)
{
if (loosewrite_ && crc32c != crc32prev)
{
// log that we chceked if the tag was matching the previous data
TRACE(Warn, CRCMismatchError(bavail, p2, crc32prev, crc32v) << " (loose match, still trying)");
if (crc32c == crc32v)
{
TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p2) << " of file " << fn_);
lastpageval = crc32c;
return 0;
}
TRACE(Warn, CRCMismatchError(bavail, p2, crc32c, crc32v));
}
else
{
TRACE(Warn, CRCMismatchError(bavail, p2, crc32prev, crc32v));
}
return -EDOM;
}
lastpageval = crc32c;
return 0;
}
XrdOssCsiCrcUtils::crc32c_combine
static uint32_t crc32c_combine(uint32_t crc1, uint32_t crc2, size_t len2)
Definition XrdOssCsiCrcUtils.hh:49

References XrdOucCRC::Calc32C(), crc32c(), CRCMismatchError(), CrcUtils, EPNAME, fn_, fullread(), loosewrite_, PageReadError(), XrdSys::PageSize, TagsReadError(), TRACE, and ts_.

Referenced by StoreRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## StoreRangeUnaligned_preblock()

int XrdOssCsiPages::StoreRangeUnaligned_preblock | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | size_t | blen,
|  | off_t | offset,
|  | off_t | trackinglen,
|  | const uint32_t * | csvec,
|  | uint32_t & | prepageval )

protected

Definition at line 148 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("StoreRangeUnaligned_preblock");
const off_t p1 = offset / XrdSys::PageSize;
const size_t p1_off = offset % XrdSys::PageSize;
const off_t tracked_page = trackinglen / XrdSys::PageSize;
const size_t tracked_off = trackinglen % XrdSys::PageSize;
if (p1 > tracked_page)
{
// the start of will have a number of implied zero bytes
uint32_t crc32c = CrcUtils.crc32c_extendwith_zero(0u, p1_off);
if (csvec)
{
crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
}
else
{
crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
}
prepageval = crc32c;
return 0;
}
// we're appending, or appending within the last page after a gap of zeros
if (p1 == tracked_page && p1_off >= tracked_off)
{
// appending: with or without some implied zeros.
// zero initialised value may be used
uint32_t crc32v = 0;
if (tracked_off > 0)
{
const ssize_t rret = ts_->ReadTags(&crc32v, p1, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(p1, 1, rret) << " (append)");
return rret;
}
}
uint32_t crc32c = 0;
// only do the loosewrite extending check one time for the page which was the
// last page according to the trackinglen at time the check was configured (open or size-resync).
// don't do the check every time because it needs an extra read compared to the non loose case;
// checklastpg_ is checked and modified here, but is protected from concurrent
// access because of the condition that p1==lastpgforloose_
if (loosewrite_ && p1==lastpgforloose_ && checklastpg_)
{
checklastpg_ = false;
uint8_t b[XrdSys::PageSize];
// this will reissue read() until eof, or tracked_off bytes read but accept up to PageSize
const ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * p1, XrdSys::PageSize, tracked_off);
if (rlen<0)
{
TRACE(Warn, PageReadError(tracked_off, p1, rlen));
return rlen;
}
memset(&b[rlen], 0, XrdSys::PageSize - rlen);
// in the loose-write mode, the new crc is based on the crc of data
// read from file up to p1_off, not on the previously stored tag.
// However must check if the data read were consistent with stored tag (crc32v)
uint32_t crc32x = XrdOucCRC::Calc32C(b, tracked_off, 0u);
crc32c = XrdOucCRC::Calc32C(&b[tracked_off], p1_off-tracked_off, crc32x);
do
{
if (static_cast<size_t>(rlen) == tracked_off)
{
// this is the expected match
if (tracked_off==0 || crc32x == crc32v) break;
}
// any bytes on disk beyond p1_off+blan would not be included in the new crc.
// if tracked_off==0 we have no meaningful crc32v value.
if ((tracked_off>0 || p1_off==0) && static_cast<size_t>(rlen) <= p1_off+blen)
{
if (tracked_off != 0)
{
TRACE(Warn, CRCMismatchError(tracked_off, p1, crc32x, crc32v) << " (loose match, still trying)");
}
// there was no tag recorded for the page, and we're completely overwriting anything on disk in the page
if (tracked_off==0)
{
TRACE(Warn, "Recovered page with no tag at offset " << (XrdSys::PageSize * p1) <<
" of file " << fn_ << " rlen=" << rlen << " (append)");
break;
}
if (static_cast<size_t>(rlen) != tracked_off && rlen>0)
{
crc32x = XrdOucCRC::Calc32C(b, rlen, 0u);
if (crc32x == crc32v)
{
TRACE(Warn, "Recovered page at offset " << (XrdSys::PageSize * p1)+p1_off << " of file " << fn_ << " (append)");
break;
}
TRACE(Warn, CRCMismatchError(rlen, p1, crc32x, crc32v) << " (loose match, still trying)");
}
memcpy(&b[p1_off], buff, blen);
crc32x = XrdOucCRC::Calc32C(b, p1_off+blen, 0u);
if (crc32x == crc32v)
{
TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p1)+p1_off <<
" of file " << fn_ << " (append)");
break;
}
TRACE(Warn, CRCMismatchError(p1_off+blen, p1, crc32x, crc32v) << " (append)");
}
else
{
if (tracked_off>0)
{
TRACE(Warn, CRCMismatchError(tracked_off, p1, crc32x, crc32v) << " (append)");
}
else
{
TRACE(Warn, "Unexpected content, write at page at offset " << (XrdSys::PageSize * p1) <<
" of file " << fn_ << ", offset-in-page=" << p1_off << " rlen=" << rlen << " (append)");
}
}
return -EDOM;
} while(0);
}
else
{
// non-loose case;
// can recalc crc with new data without re-reading existing partial block's data
const size_t nz = p1_off - tracked_off;
crc32c = CrcUtils.crc32c_extendwith_zero(crc32v, nz);
}
// crc32c is crc up to p1_off. Now add the user's data.
if (csvec)
{
crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
}
else
{
crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
}
prepageval = crc32c;
return 0;
}
const size_t bavail = (p1==tracked_page) ? tracked_off : XrdSys::PageSize;
// assert we're overwriting some (or all) of the previous data (other case was above)
assert(p1_off < bavail);
// case p1_off==0 && blen>=bavail is either handled by aligned case (p1==tracked_page)
// or not sent to preblock, so will need to read some preexisting data
assert(p1_off !=0 || blen<bavail);
uint8_t b[XrdSys::PageSize];
uint32_t crc32v;
ssize_t rret = ts_->ReadTags(&crc32v, p1, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(p1, 1, rret) << " (overwrite)");
return rret;
}
// in either loosewrite or non-loosewrite a read-modify-write sequence is done and the
// final crc is that of the modified block. The difference between loose and non-loose
// case if that the looser checks are done on the block.
//
// in either case there are implicit verification(s) (e.g. pgWrite may return EDOM without Verify requested)
// as it's not clear if there is a meaningful way to crc a mismatching page during a partial overwrite
if (loosewrite_)
{
// this will reissue read() until eof, or bavail bytes read but accept up to PageSize
const ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * p1, XrdSys::PageSize, bavail);
if (rlen<0)
{
TRACE(Warn, PageReadError(bavail, p1, rlen));
return rlen;
}
memset(&b[rlen], 0, XrdSys::PageSize - rlen);
do
{
uint32_t crc32c = XrdOucCRC::Calc32C(b, bavail, 0U);
// this is the expected case
if (static_cast<size_t>(rlen) == bavail && crc32c == crc32v) break;
// after this write there will be nothing changed between p1_off+blen
// and bavail; if there is nothing on disk in this range it will not
// be added by the write. So don't try to match crc with implied zero
// in this range. Beyond bavail bytes on disk will not be included
// in the new crc.
const size_t rmin = (p1_off+blen < bavail) ? bavail : 0;
if (static_cast<size_t>(rlen) >= rmin && static_cast<size_t>(rlen)<=bavail)
{
if (crc32c == crc32v)
{
TRACE(Warn, "Recovered page at offset " << (XrdSys::PageSize * p1) << " of file " << fn_ << " (overwrite)");
break;
}
TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v) << " (loose match, still trying)");
if (static_cast<size_t>(rlen) != bavail && rlen > 0)
{
crc32c = XrdOucCRC::Calc32C(b, rlen, 0U);
if (crc32c == crc32v)
{
TRACE(Warn, "Recovered page (2) at offset " << (XrdSys::PageSize * p1) << " of file " << fn_ << " (overwrite)");
break;
}
TRACE(Warn, CRCMismatchError(rlen, p1, crc32c, crc32v) << " (loose match, still trying)");
}
memcpy(&b[p1_off], buff, blen);
const size_t vl = std::max(bavail, p1_off+blen);
crc32c = XrdOucCRC::Calc32C(b, vl, 0U);
if (crc32c == crc32v)
{
TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p1)+p1_off << " of file " << fn_ << " (overwrite)");
break;
}
TRACE(Warn, CRCMismatchError(vl, p1, crc32c, crc32v) << " (overwrite)");
}
else
{
TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v) << " (overwrite)");
}
return -EDOM;
} while(0);
}
else
{
// non-loose case
rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize * p1, bavail);
if (rret<0)
{
TRACE(Warn, PageReadError(bavail, p1, rret));
return rret;
}
const uint32_t crc32c = XrdOucCRC::Calc32C(b, bavail, 0U);
if (crc32v != crc32c)
{
TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v));
return -EDOM;
}
}
uint32_t crc32c = XrdOucCRC::Calc32C(b, p1_off, 0U);
if (csvec)
{
crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
}
else
{
crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
}
if (p1_off+blen < bavail)
{
const uint32_t cl = XrdOucCRC::Calc32C(&b[p1_off+blen], bavail-p1_off-blen, 0U);
crc32c = CrcUtils.crc32c_combine(crc32c, cl, bavail-p1_off-blen);
}
prepageval = crc32c;
return 0;
}
XrdOssCsiCrcUtils::crc32c_extendwith_zero
static uint32_t crc32c_extendwith_zero(uint32_t crc, size_t len)
Definition XrdOssCsiCrcUtils.hh:107

References XrdOucCRC::Calc32C(), checklastpg_, crc32c(), CRCMismatchError(), CrcUtils, EPNAME, fn_, fullread(), lastpgforloose_, loosewrite_, maxread(), PageReadError(), XrdSys::PageSize, TagsReadError(), TRACE, and ts_.

Referenced by StoreRangeUnaligned().

Here is the call graph for this function:
Here is the caller graph for this function:

## TagsReadError()

std::string XrdOssCsiPages::TagsReadError | ( | off_t | start,
|  | size_t | n,
|  | int | ret )

inline protected

Definition at line 186 of file XrdOssCsiPages.hh.

{
char buf[256];
snprintf(buf, sizeof(buf),
"error %d while reading crc32c values for pages [0x%" PRIx64":0x%" PRIx64 "] for file ",
ret, (uint64_t)start, (uint64_t)(start + n - 1));
return buf + fn_;
}

References fn_.

Referenced by BasicConsistencyCheck(), FetchRangeAligned(), FetchRangeUnaligned(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), truncate(), and UpdateRangeHoleUntilPage().

Here is the caller graph for this function:

## TagsWriteError()

std::string XrdOssCsiPages::TagsWriteError | ( | off_t | start,
|  | size_t | n,
|  | int | ret )

inline protected

Definition at line 195 of file XrdOssCsiPages.hh.

{
char buf[256];
snprintf(buf, sizeof(buf),
"error %d while writing crc32c values for pages [0x%" PRIx64":0x%" PRIx64 "] for file ",
ret, (uint64_t)start, (uint64_t)(start + n - 1));
return buf + fn_;
}

References fn_.

Referenced by apply_sequential_aligned_modify(), BasicConsistencyCheck(), StoreRangeUnaligned(), truncate(), and UpdateRangeHoleUntilPage().

Here is the caller graph for this function:

## TrackedSizeRelease()

void XrdOssCsiPages::TrackedSizeRelease | ( |  | ) |

Definition at line 158 of file XrdOssCsiPages.cc.

{
XrdSysCondVarHelper lck(&tscond_);
assert(tsforupdate_ == true);
tsforupdate_ = false;
tscond_.Broadcast();
}

References tscond_, and tsforupdate_.

Referenced by LockTrackinglen().

Here is the caller graph for this function:

## TrackedSizesGet()

int XrdOssCsiPages::TrackedSizesGet | ( | XrdOssCsiPages::Sizes_t & | rsizes,
|  | bool | forupdate )

Definition at line 109 of file XrdOssCsiPages.cc.

{
if (hasMissingTags_) return -ENOENT;
XrdSysCondVarHelper lck(&tscond_);
while (tsforupdate_)
{
tscond_.Wait();
}
off_t tagsize = ts_->GetTrackedTagSize();
off_t datasize = ts_->GetTrackedDataSize();
if (forupdate)
{
tsforupdate_ = true;
}
rsizes = std::make_pair(tagsize,datasize);
return 0;
}

References hasMissingTags_, ts_, tscond_, and tsforupdate_.

Referenced by XrdOssCsiFile::Fstat(), and LockTrackinglen().

Here is the caller graph for this function:

## truncate()

int XrdOssCsiPages::truncate | ( | XrdOssDF * | fd,
|  | off_t | len,
|  | XrdOssCsiRangeGuard & | rg )

Definition at line 543 of file XrdOssCsiPages.cc.

{
EPNAME("truncate");
if (len<0) return -EINVAL;
// nothing to truncate if there is no tag file
if (hasMissingTags_) return 0;
const Sizes_t sizes = rg.getTrackinglens();
const off_t trackinglen = sizes.first;
const off_t p_until = len / XrdSys::PageSize;
const size_t p_off = len % XrdSys::PageSize;
if (len>trackinglen)
{
int ret = UpdateRangeHoleUntilPage(fd,p_until,sizes);
if (ret<0)
{
TRACE(Warn, "Error updating tags for holes, error=" << ret);
return ret;
}
}
if (len != trackinglen && p_off != 0)
{
const off_t tracked_page = trackinglen / XrdSys::PageSize;
const size_t tracked_off = trackinglen % XrdSys::PageSize;
size_t toread = tracked_off;
if (len>trackinglen)
{
if (p_until != tracked_page) toread = 0;
}
else
{
if (p_until != tracked_page) toread = XrdSys::PageSize;
}
uint8_t b[XrdSys::PageSize];
if (toread>0)
{
ssize_t rret = XrdOssCsiPages::fullread(fd, b, p_until*XrdSys::PageSize, toread);
if (rret<0)
{
TRACE(Warn, PageReadError(toread, p_until, rret));
return rret;
}
const uint32_t crc32c = XrdOucCRC::Calc32C(b, toread, 0U);
uint32_t crc32v;
rret = ts_->ReadTags(&crc32v, p_until, 1);
if (rret<0)
{
TRACE(Warn, TagsReadError(p_until, 1, rret));
return rret;
}
if (crc32v != crc32c)
{
TRACE(Warn, CRCMismatchError(toread, p_until, crc32c, crc32v));
return -EDOM;
}
}
if (p_off > toread)
{
memset(&b[toread],0,p_off-toread);
}
const uint32_t crc32c = XrdOucCRC::Calc32C(b, p_off, 0U);
const ssize_t wret = ts_->WriteTags(&crc32c, p_until, 1);
if (wret < 0)
{
TRACE(Warn, TagsWriteError(p_until, 1, wret));
return wret;
}
}
LockTruncateSize(len,true);
rg.unlockTrackinglen();
return 0;
}
XrdOssCsiPages::LockTruncateSize
int LockTruncateSize(off_t, bool)
Definition XrdOssCsiPages.cc:146

References XrdOucCRC::Calc32C(), crc32c(), CRCMismatchError(), EPNAME, fullread(), XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockTruncateSize(), PageReadError(), XrdSys::PageSize, TagsReadError(), TagsWriteError(), TRACE, ts_, XrdOssCsiRangeGuard::unlockTrackinglen(), and UpdateRangeHoleUntilPage().

Referenced by XrdOssCsiFile::Ftruncate().

Here is the call graph for this function:
Here is the caller graph for this function:

## UpdateRange()

int XrdOssCsiPages::UpdateRange | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | XrdOssCsiRangeGuard & | rg )

Definition at line 169 of file XrdOssCsiPages.cc.

{
if (offset<0)
{
return -EINVAL;
}
if (blen == 0)
{
return 0;
}
// if the tag file is missing we don't need to store anything
if (hasMissingTags_)
{
return 0;
}
// update of file were checksums are based on the file data suppplied: as there's no separate
// source of checksum information mark this file as having unverified checksums
LockMakeUnverified();
const Sizes_t sizes = rg.getTrackinglens();
const off_t trackinglen = sizes.first;
if (offset+blen > static_cast<size_t>(trackinglen))
{
LockSetTrackedSize(offset+blen);
rg.unlockTrackinglen();
}
int ret;
if ((offset % XrdSys::PageSize) != 0 ||
(offset+blen < static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0) ||
((trackinglen % XrdSys::PageSize) !=0 && offset > trackinglen))
{
ret = UpdateRangeUnaligned(fd, buff, offset, blen, sizes);
}
else
{
ret = UpdateRangeAligned(buff, offset, blen, sizes);
}
return ret;
}
XrdOssCsiPages::UpdateRangeAligned
int UpdateRangeAligned(const void *, off_t, size_t, const Sizes_t &)
Definition XrdOssCsiPages.cc:485
XrdOssCsiPages::UpdateRangeUnaligned
int UpdateRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
Definition XrdOssCsiPagesUnaligned.cc:137

References XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockMakeUnverified(), LockSetTrackedSize(), XrdSys::PageSize, XrdOssCsiRangeGuard::unlockTrackinglen(), UpdateRangeAligned(), and UpdateRangeUnaligned().

Referenced by XrdOssCsiFile::Write(), and XrdOssCsiFile::WriteV().

Here is the call graph for this function:
Here is the caller graph for this function:

## UpdateRangeAligned()

int XrdOssCsiPages::UpdateRangeAligned | ( | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes )

protected

Definition at line 485 of file XrdOssCsiPages.cc.

{
return StoreRangeAligned(buff, offset, blen, sizes, NULL);
}

References StoreRangeAligned().

Referenced by UpdateRange().

Here is the call graph for this function:
Here is the caller graph for this function:

## UpdateRangeHoleUntilPage()

int XrdOssCsiPages::UpdateRangeHoleUntilPage | ( | XrdOssDF * | fd,
|  | off_t | until,
|  | const Sizes_t & | sizes )

protected

Definition at line 52 of file XrdOssCsiPagesUnaligned.cc.

{
EPNAME("UpdateRangeHoleUntilPage");
static const uint32_t crczero = CrcUtils.crc32c_extendwith_zero(0u, XrdSys::PageSize);
static const std::vector<uint32_t> crc32Vec(stsize_, crczero);
static const std::vector<uint32_t> crc32VecZ(stsize_, 0u);
const off_t trackinglen = sizes.first;
const off_t tracked_page = trackinglen / XrdSys::PageSize;
if (until <= tracked_page) return 0;
const size_t tracked_off = trackinglen % XrdSys::PageSize;
// if last tracked page is before page "until" extend it
if (tracked_off>0)
{
if (fd == NULL)
{
TRACE(Warn, "Unexpected partially filled last page " << fn_);
return -EDOM;
}
uint32_t prevtag;
const ssize_t rret = ts_->ReadTags(&prevtag, tracked_page, 1);
if (rret < 0)
{
TRACE(Warn, TagsReadError(tracked_page, 1, rret));
return rret;
}
// extend prevtag up to PageSize. If there is a mismatch it will only be
// discovered during a later read (but this saves a read now).
const uint32_t crc32c = CrcUtils.crc32c_extendwith_zero(prevtag, XrdSys::PageSize - tracked_off);
const ssize_t wret = ts_->WriteTags(&crc32c, tracked_page, 1);
if (wret < 0)
{
TRACE(Warn, TagsWriteError(tracked_page, 1, wret) << " (prev)");
return wret;
}
}
const off_t nAllEmpty = (tracked_off>0) ? (until - tracked_page - 1) : (until - tracked_page);
const off_t firstEmpty = (tracked_off>0) ? (tracked_page + 1) : tracked_page;
off_t towrite = nAllEmpty;
off_t nwritten = 0;
while(towrite>0)
{
off_t wblks = 0;
const uint32_t *wpointer = 0;
// unless we are configured with nofill (!nofill means writeHoles_ is true), we fill
// in the crc for an implied zero block. Reads from this hole will be valid, which
// is the usual behavior for sparse files. If nofill is configured it is assumed that the
// user knows the application does not expect to make such reads, so we set a zero tag
// so there will be a mismatch to catch the situation; we write the zero tag explicitly
// so that the tag file itself is not sparse.
if (writeHoles_)
{
wblks = (off_t)crc32Vec.size();
wpointer = &crc32Vec[0];
}
else
{
wblks = (off_t)crc32VecZ.size();
wpointer = &crc32VecZ[0];
}
const size_t nw = std::min(towrite, wblks);
const ssize_t wret = ts_->WriteTags(wpointer, firstEmpty+nwritten, nw);
if (wret<0)
{
TRACE(Warn, TagsWriteError(firstEmpty+nwritten, nw, wret) << " (new)");
return wret;
}
towrite -= wret;
nwritten += wret;
}
return 0;
}

References crc32c(), CrcUtils, EPNAME, fn_, XrdSys::PageSize, stsize_, TagsReadError(), TagsWriteError(), TRACE, ts_, and writeHoles_.

Referenced by StoreRangeAligned(), StoreRangeUnaligned(), and truncate().

Here is the call graph for this function:
Here is the caller graph for this function:

## UpdateRangeUnaligned()

int XrdOssCsiPages::UpdateRangeUnaligned | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes )

protected

Definition at line 137 of file XrdOssCsiPagesUnaligned.cc.

{
return StoreRangeUnaligned(fd, buff, offset, blen, sizes, NULL);
}

References StoreRangeUnaligned().

Referenced by UpdateRange().

Here is the call graph for this function:
Here is the caller graph for this function:

## VerificationStatus()

int XrdOssCsiPages::VerificationStatus | ( |  | ) |

Definition at line 759 of file XrdOssCsiPages.cc.

{
if (hasMissingTags_)
{
return 0;
}
bool iv;
{
XrdSysCondVarHelper lck(&tscond_);
iv = ts_->IsVerified();
}
if (iv)
{
return XrdOss::PF_csVer;
}
return XrdOss::PF_csVun;
}
XrdOss::PF_csVer
static const int PF_csVer
verified file checksums present
Definition XrdOss.hh:778
XrdOss::PF_csVun
static const int PF_csVun
unverified file checksums present
Definition XrdOss.hh:779

References hasMissingTags_, XrdOss::PF_csVer, XrdOss::PF_csVun, ts_, and tscond_.

Referenced by XrdOssCsiFile::VerificationStatus().

Here is the caller graph for this function:

## VerifyRange()

int XrdOssCsiPages::VerifyRange | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | XrdOssCsiRangeGuard & | rg )

Definition at line 220 of file XrdOssCsiPages.cc.

{
EPNAME("VerifyRange");
if (offset<0)
{
return -EINVAL;
}
// if the tag file is missing we don't verify anything
if (hasMissingTags_)
{
return 0;
}
const Sizes_t sizes = rg.getTrackinglens();
const off_t trackinglen = sizes.first;
if (offset >= trackinglen && blen == 0)
{
return 0;
}
if (blen == 0)
{
// if offset is before the tracked len we should not be requested to verify zero bytes:
// the file may have been truncated
TRACE(Warn, "Verify request for zero bytes " << fn_ << ", file may be truncated");
return -EDOM;
}
if (offset+blen > static_cast<size_t>(trackinglen))
{
TRACE(Warn, "Verify request for " << (offset+blen-trackinglen) << " bytes from " << fn_ << " beyond tracked length");
return -EDOM;
}
int vret;
if ((offset % XrdSys::PageSize) != 0 || (offset+blen != static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0))
{
vret = VerifyRangeUnaligned(fd, buff, offset, blen, sizes);
}
else
{
vret = VerifyRangeAligned(buff, offset, blen, sizes);
}
return vret;
}
XrdOssCsiPages::VerifyRangeAligned
int VerifyRangeAligned(const void *, off_t, size_t, const Sizes_t &)
Definition XrdOssCsiPages.cc:448
XrdOssCsiPages::VerifyRangeUnaligned
int VerifyRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
Definition XrdOssCsiPagesUnaligned.cc:628

References EPNAME, fn_, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, XrdSys::PageSize, TRACE, VerifyRangeAligned(), and VerifyRangeUnaligned().

Referenced by XrdOssCsiFile::Read(), XrdOssCsiFile::ReadRaw(), and XrdOssCsiFile::ReadV().

Here is the call graph for this function:
Here is the caller graph for this function:

## VerifyRangeAligned()

int XrdOssCsiPages::VerifyRangeAligned | ( | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes )

protected

Definition at line 448 of file XrdOssCsiPages.cc.

{
return FetchRangeAligned(buff,offset,blen,sizes,NULL,XrdOssDF::Verify);
}

References FetchRangeAligned(), and XrdOssDF::Verify.

Referenced by VerifyRange().

Here is the call graph for this function:
Here is the caller graph for this function:

## VerifyRangeUnaligned()

int XrdOssCsiPages::VerifyRangeUnaligned | ( | XrdOssDF * | fd,
|  | const void * | buff,
|  | off_t | offset,
|  | size_t | blen,
|  | const Sizes_t & | sizes )

protected

Definition at line 628 of file XrdOssCsiPagesUnaligned.cc.

{
return FetchRangeUnaligned(fd, buff, offset, blen, sizes, NULL, XrdOssDF::Verify);
}

References FetchRangeUnaligned(), and XrdOssDF::Verify.

Referenced by VerifyRange().

Here is the call graph for this function:
Here is the caller graph for this function:

## Member Data Documentation

## allowMissingTags_

bool XrdOssCsiPages::allowMissingTags_

protected

Definition at line 84 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), and Open().

## checklastpg_

bool XrdOssCsiPages::checklastpg_

protected

Definition at line 102 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), BasicConsistencyCheck(), and StoreRangeUnaligned_preblock().

## disablePgExtend_

bool XrdOssCsiPages::disablePgExtend_

protected

Definition at line 85 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), and StoreRange().

## fn_

const std::string XrdOssCsiPages::fn_

protected

Definition at line 96 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), ByteMismatchError(), CRCMismatchError(), FetchRange(), Open(), PageReadError(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), TagsReadError(), TagsWriteError(), UpdateRangeHoleUntilPage(), and VerifyRange().

## hasMissingTags_

bool XrdOssCsiPages::hasMissingTags_

protected

Definition at line 86 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), Close(), FetchRange(), Flush(), Fsync(), LockResetSizes(), LockTrackinglen(), Open(), StoreRange(), TrackedSizesGet(), truncate(), UpdateRange(), VerificationStatus(), and VerifyRange().

## lastpgforloose_

off_t XrdOssCsiPages::lastpgforloose_

protected

Definition at line 101 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), BasicConsistencyCheck(), and StoreRangeUnaligned_preblock().

## loosewrite_

bool XrdOssCsiPages::loosewrite_

protected

Definition at line 89 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), BasicConsistencyCheck(), LockResetSizes(), Open(), StoreRangeUnaligned_postblock(), and StoreRangeUnaligned_preblock().

## loosewriteConfigured_

const bool XrdOssCsiPages::loosewriteConfigured_

protected

Definition at line 88 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), LockResetSizes(), and Open().

## rangeaddmtx_

XrdSysMutex XrdOssCsiPages::rangeaddmtx_

protected

Definition at line 81 of file XrdOssCsiPages.hh.

Referenced by LockTrackinglen().

## ranges_

XrdOssCsiRanges XrdOssCsiPages::ranges_

protected

Definition at line 82 of file XrdOssCsiPages.hh.

Referenced by LockTrackinglen().

## rdonly_

bool XrdOssCsiPages::rdonly_

protected

Definition at line 87 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), BasicConsistencyCheck(), IsReadOnly(), and Open().

## stsize_

const size_t XrdOssCsiPages::stsize_ = 1024

static protected

Definition at line 204 of file XrdOssCsiPages.hh.

Referenced by apply_sequential_aligned_modify(), FetchRangeAligned(), FetchRangeUnaligned(), and UpdateRangeHoleUntilPage().

## tident

const char* XrdOssCsiPages::tident

protected

Definition at line 98 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages().

## tident_

const std::string XrdOssCsiPages::tident_

protected

Definition at line 97 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages().

## ts_

std::unique_ptr<XrdOssCsiTagstore> XrdOssCsiPages::ts_

protected

Definition at line 80 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), apply_sequential_aligned_modify(), BasicConsistencyCheck(), Close(), FetchRangeAligned(), FetchRangeUnaligned(), Flush(), Fsync(), LockMakeUnverified(), LockResetSizes(), LockSetTrackedSize(), LockTruncateSize(), Open(), StoreRangeUnaligned(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), TrackedSizesGet(), truncate(), UpdateRangeHoleUntilPage(), and VerificationStatus().

## tscond_

XrdSysCondVar XrdOssCsiPages::tscond_

protected

Definition at line 91 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), LockMakeUnverified(), LockResetSizes(), LockSetTrackedSize(), LockTruncateSize(), TrackedSizeRelease(), TrackedSizesGet(), and VerificationStatus().

## tsforupdate_

bool XrdOssCsiPages::tsforupdate_

protected

Definition at line 92 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), TrackedSizeRelease(), and TrackedSizesGet().

## writeHoles_

bool XrdOssCsiPages::writeHoles_

protected

Definition at line 83 of file XrdOssCsiPages.hh.

Referenced by XrdOssCsiPages(), and UpdateRangeHoleUntilPage().

The documentation for this class was generated from the following files:

- XrdOssCsiPages.hh
- XrdOssCsiPages.cc
- XrdOssCsiPagesUnaligned.cc