Loading...
Searching...
No Matches
XrdClUtils.cc
Go to the documentation of this file.
//------------------------------------------------------------------------------
// Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN)
// Author: Lukasz Janyst <ljanyst@cern.ch>
//------------------------------------------------------------------------------
// XRootD is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// XRootD is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with XRootD. If not, see <http://www.gnu.org/licenses/>.
//------------------------------------------------------------------------------
#include "XrdCl/XrdClUtils.hh"
#include "XrdCl/XrdClFileSystem.hh"
#include "XrdCl/XrdClDefaultEnv.hh"
#include "XrdCl/XrdClConstants.hh"
#include "XrdCl/XrdClCheckSumManager.hh"
#include "XrdCl/XrdClRedirectorRegistry.hh"
#include "XrdCl/XrdClMessage.hh"
#include "XrdCl/XrdClOptimizers.hh"
#include "XrdNet/XrdNetAddr.hh"
#include <algorithm>
#include <iomanip>
#include <sstream>
#include <fstream>
#include <functional>
#include <cctype>
#include <locale>
#include <map>
#include <string>
#include <set>
#include <cctype>
#include <random>
#include <chrono>
#include <sys/types.h>
#include <dirent.h>
#if __cplusplus < 201103L
#include <ctime>
#endif
namespace
{
bool isNotSpace( char c )
{
return c != ' ';
}
//----------------------------------------------------------------------------
// Ordering function for sorting IP addresses
//----------------------------------------------------------------------------
struct PreferIPv6
{
bool operator() ( const XrdNetAddr &l, const XrdNetAddr &r )
{
bool rIsIPv4 = false;
if( r.isIPType( XrdNetAddrInfo::IPv4 ) ||
(r.isIPType( XrdNetAddrInfo::IPv6 ) && r.isMapped()) )
rIsIPv4 = true;
if( l.isIPType( XrdNetAddrInfo::IPv6 ) && rIsIPv4 )
return true;
return false;
}
};
}
namespace XrdCl
{
//----------------------------------------------------------------------------
// Get a parameter either from the environment or URL
//----------------------------------------------------------------------------
int Utils::GetIntParameter( const URL &url,
const std::string &name,
int defaultVal )
{
Env *env = DefaultEnv::GetEnv();
int value = defaultVal;
char *endPtr;
URL::ParamsMap::const_iterator it;
env->GetInt( name, value );
it = url.GetParams().find( std::string("XrdCl.") + name );
if( it != url.GetParams().end() )
{
int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
if( !*endPtr )
value = urlValue;
}
return value;
}
//----------------------------------------------------------------------------
// Get a parameter either from the environment or URL
//----------------------------------------------------------------------------
std::string Utils::GetStringParameter( const URL &url,
const std::string &name,
const std::string &defaultVal )
{
Env *env = DefaultEnv::GetEnv();
std::string value = defaultVal;
URL::ParamsMap::const_iterator it;
env->GetString( name, value );
it = url.GetParams().find( std::string("XrdCl.") + name );
if( it != url.GetParams().end() )
value = it->second;
return value;
}
//----------------------------------------------------------------------------
// Interpret a string as address type, default to IPAll
//----------------------------------------------------------------------------
Utils::AddressType Utils::String2AddressType( const std::string &addressType )
{
if( addressType == "IPv6" )
return IPv6;
else if( addressType == "IPv4" )
return IPv4;
else if( addressType == "IPv4Mapped6" )
return IPv4Mapped6;
else if( addressType == "IPAll" )
return IPAll;
else
return IPAuto;
}
//----------------------------------------------------------------------------
// Resolve IP addresses
//----------------------------------------------------------------------------
Status Utils::GetHostAddresses( std::vector<XrdNetAddr> &addresses,
const URL &url,
Utils::AddressType type )
{
Log *log = DefaultEnv::GetLog();
const char *err = 0;
int ordn;
//--------------------------------------------------------------------------
// Resolve all the addresses
//--------------------------------------------------------------------------
std::ostringstream o; o << url.GetHostName() << ":" << url.GetPort();
XrdNetUtils::AddrOpts opts;
if( type == IPv6 ) opts = XrdNetUtils::onlyIPv6;
else if( type == IPv4 ) opts = XrdNetUtils::onlyIPv4;
else if( type == IPv4Mapped6 ) opts = XrdNetUtils::allV4Map;
else if( type == IPAll ) opts = XrdNetUtils::allIPMap;
else opts = XrdNetUtils::prefAuto;
//--------------------------------------------------------------------------
// Check what are the preferences IPv6 or IPv4
//--------------------------------------------------------------------------
int preferIPv4 = DefaultPreferIPv4;
DefaultEnv::GetEnv()->GetInt( "PreferIPv4", preferIPv4 );
//--------------------------------------------------------------------------
// Partition the addresses according to the preferences
//
// The preferred IP family goes to the back as it is easier to remove
// items from the back of the vector
//--------------------------------------------------------------------------
opts |= (preferIPv4 ? XrdNetUtils::order64 : XrdNetUtils::order46);
//--------------------------------------------------------------------------
// Now get all of the properly partitioned addresses; ordn will hold the
// number of non-preferred addresses at the front of the table.
//--------------------------------------------------------------------------
err = XrdNetUtils::GetAddrs( o.str(), addresses, &ordn, opts );
if( err )
{
log->Error( UtilityMsg, "Unable to resolve %s: %s", o.str().c_str(),
err );
return Status( stError, errInvalidAddr );
}
if( addresses.size() == 0 )
{
log->Error( UtilityMsg, "No addresses for %s were found",
o.str().c_str() );
return Status( stError, errInvalidAddr );
}
//--------------------------------------------------------------------------
// Shuffle each partition
//--------------------------------------------------------------------------
int ipNoShuffle = DefaultIPNoShuffle;
Env *env = DefaultEnv::GetEnv();
env->GetInt( "IPNoShuffle", ipNoShuffle );
if( !ipNoShuffle )
{
#if __cplusplus < 201103L
// initialize the random generator only once
static struct only_once_t
{
only_once_t()
{
std::srand ( unsigned ( std::time(0) ) );
}
} only_once;
std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
std::random_shuffle( addresses.begin() + ordn, addresses.end() );
#else
static std::default_random_engine rand_engine(
std::chrono::system_clock::now().time_since_epoch().count() );
std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
#endif
}
//--------------------------------------------------------------------------
// Return status as the result is already in the output parameter
//--------------------------------------------------------------------------
return Status();
}
//----------------------------------------------------------------------------
// Log all the addresses on the list
//----------------------------------------------------------------------------
void Utils::LogHostAddresses( Log *log,
uint64_t type,
const std::string &hostId,
std::vector<XrdNetAddr> &addresses )
{
std::string addrStr;
std::vector<XrdNetAddr>::iterator it;
for( it = addresses.begin(); it != addresses.end(); ++it )
{
char nameBuff[256];
it->Format( nameBuff, 256, XrdNetAddrInfo::fmtAdv6 );
addrStr += nameBuff;
addrStr += ", ";
}
addrStr.erase( addrStr.length()-2, 2 );
log->Debug( type, "[%s] Found %zu address(es): %s",
hostId.c_str(), addresses.size(), addrStr.c_str() );
}
//----------------------------------------------------------------------------
// Convert timestamp to a string
//----------------------------------------------------------------------------
std::string Utils::TimeToString( time_t timestamp )
{
char now[30];
tm tsNow;
time_t ttNow = timestamp;
localtime_r( &ttNow, &tsNow );
strftime( now, 30, "%Y-%m-%d %H:%M:%S %z", &tsNow );
return now;
}
//----------------------------------------------------------------------------
// Get the elapsed microseconds between two timevals
//----------------------------------------------------------------------------
uint64_t Utils::GetElapsedMicroSecs( timeval start, timeval end )
{
uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
return endUSec-startUSec;
}
//----------------------------------------------------------------------------
// Get remote checksum
//----------------------------------------------------------------------------
XRootDStatus Utils::GetRemoteCheckSum( std::string &checkSum,
const std::string &checkSumType,
const URL &url )
{
FileSystem *fs = new FileSystem( url );
// add the 'cks.type' cgi tag in order to
// select the proper checksum type in case
// the server supports more than one checksum
size_t pos = url.GetPath().find( '?' );
std::string cksPath = url.GetPath() + ( pos == std::string::npos ? '?' : '&' ) + "cks.type=" + checkSumType;
Buffer arg; arg.FromString( cksPath );
Buffer *cksResponse = 0;
XRootDStatus st;
Log *log = DefaultEnv::GetLog();
st = fs->Query( QueryCode::Checksum, arg, cksResponse );
delete fs;
if( !st.IsOK() )
{
std::string msg = st.GetErrorMessage();
msg += " Got an error while querying the checksum!";
st.SetErrorMessage( msg );
return st;
}
if( !cksResponse )
return XRootDStatus( stError, errInternal, 0, "Got invalid response while querying the checksum!" );
std::vector<std::string> elems;
Utils::splitString( elems, cksResponse->ToString(), " " );
delete cksResponse;
if( elems.size() != 2 )
return XRootDStatus( stError, errInvalidResponse, 0, "Got invalid response while querying the checksum!" );
if( elems[0] != checkSumType )
return XRootDStatus( stError, errCheckSumError );
checkSum = elems[0] + ":";
checkSum += NormalizeChecksum( elems[0], elems[1] );
log->Dump( UtilityMsg, "Checksum for %s checksum: %s",
url.GetPath().c_str(), checkSum.c_str() );
return XRootDStatus();
}
//------------------------------------------------------------------------
// Get a checksum from local file
//------------------------------------------------------------------------
XRootDStatus Utils::GetLocalCheckSum( std::string &checkSum,
const std::string &checkSumType,
const std::string &path )
{
Log *log = DefaultEnv::GetLog();
CheckSumManager *cksMan = DefaultEnv::GetCheckSumManager();
if( !cksMan )
{
log->Error( UtilityMsg, "Unable to get the checksum manager" );
return XRootDStatus( stError, errInternal );
}
XrdCksData ckSum; ckSum.Set( checkSumType.c_str() );
bool status = cksMan->Calculate( ckSum, checkSumType, path.c_str() );
if( !status )
{
log->Error( UtilityMsg, "Error while calculating checksum for %s",
path.c_str() );
return XRootDStatus( stError, errCheckSumError );
}
char *cksBuffer = new char[265];
ckSum.Get( cksBuffer, 256 );
checkSum = checkSumType + ":";
checkSum += NormalizeChecksum( checkSumType, cksBuffer );
delete [] cksBuffer;
log->Dump( UtilityMsg, "Checksum for %s is: %s", path.c_str(),
checkSum.c_str() );
return XRootDStatus();
}
//----------------------------------------------------------------------------
// Convert bytes to a human readable string
//----------------------------------------------------------------------------
std::string Utils::BytesToString( uint64_t bytes )
{
double final = bytes;
int i = 0;
char suf[3] = { 'k', 'M', 'G' };
for( i = 0; i < 3 && final > 1024; ++i, final /= 1024 ) {};
std::ostringstream o;
o << std::setprecision(4) << final;
if( i > 0 ) o << suf[i-1];
return o.str();
}
//----------------------------------------------------------------------------
// Check if peer supports tpc
//----------------------------------------------------------------------------
XRootDStatus Utils::CheckTPC( const std::string &server, uint16_t timeout )
{
Log *log = DefaultEnv::GetLog();
log->Debug( UtilityMsg, "Checking if the data server %s supports tpc",
server.c_str() );
FileSystem sourceDSFS( server );
Buffer queryArg; queryArg.FromString( "tpc" );
Buffer *queryResponse = 0;
XRootDStatus st;
st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
timeout );
if( !st.IsOK() )
{
log->Error( UtilityMsg, "Cannot query source data server %s: %s",
server.c_str(), st.ToStr().c_str() );
st.status = stFatal;
return st;
}
if( !queryResponse )
{
log->Error( UtilityMsg, "Cannot query source data server: empty response." );
st.status = stFatal;
return st;
}
std::string answer = queryResponse->ToString();
delete queryResponse;
if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
{
log->Debug( UtilityMsg, "Third party copy not supported at: %s",
server.c_str() );
return XRootDStatus( stError, errNotSupported );
}
log->Debug( UtilityMsg, "Third party copy supported at: %s",
server.c_str() );
return XRootDStatus();
}
//------------------------------------------------------------------------
// Check if peer supports tpc / tpc lite
//------------------------------------------------------------------------
XRootDStatus Utils::CheckTPCLite( const std::string &server, uint16_t timeout )
{
Log *log = DefaultEnv::GetLog();
log->Debug( UtilityMsg, "Checking if the data server %s supports tpc / tpc lite",
server.c_str() );
FileSystem sourceDSFS( server );
Buffer queryArg; queryArg.FromString( "tpc tpcdlg" );
Buffer *queryResponse = 0;
XRootDStatus st;
st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
timeout );
if( !st.IsOK() )
{
log->Error( UtilityMsg, "Cannot query source data server %s: %s",
server.c_str(), st.ToStr().c_str() );
st.status = stFatal;
return st;
}
if( !queryResponse )
{
log->Error( UtilityMsg, "Cannot query source data server: empty response." );
st.status = stFatal;
return st;
}
std::string answer = queryResponse->ToString();
delete queryResponse;
if( answer.empty() )
{
log->Error( UtilityMsg, "Cannot query source data server: empty response." );
st.status = stFatal;
return st;
}
std::vector<std::string> resp;
Utils::splitString( resp, answer, "\n" );
if( resp.empty() || resp[0].empty() ||
!isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
{
log->Debug( UtilityMsg, "Third party copy not supported at: %s",
server.c_str() );
return XRootDStatus( stError, errNotSupported );
}
if( resp.size() == 1 || resp[1] == "tpcdlg" )
{
log->Debug( UtilityMsg, "TPC lite not supported at: %s",
server.c_str() );
return XRootDStatus( stOK, suPartial );
}
log->Debug( UtilityMsg, "TPC lite supported at: %s",
server.c_str() );
return XRootDStatus();
}
//----------------------------------------------------------------------------
// Convert the fully qualified host name to country code
//----------------------------------------------------------------------------
std::string Utils::FQDNToCC( const std::string &fqdn )
{
std::vector<std::string> el;
Utils::splitString( el, fqdn, "." );
if( el.size() < 2 )
return "us";
std::string cc = *el.rbegin();
if( cc.length() == 2 )
return cc;
return "us";
}
//----------------------------------------------------------------------------
// Get directory entries
//----------------------------------------------------------------------------
Status Utils::GetDirectoryEntries( std::vector<std::string> &entries,
const std::string &path )
{
DIR *dp = opendir( path.c_str() );
if( !dp )
return Status( stError, errOSError, errno );
dirent *dirEntry;
while( (dirEntry = readdir(dp)) != 0 )
{
std::string entryName = dirEntry->d_name;
if( !entryName.compare( 0, 2, "..") )
continue;
if( !entryName.compare( 0, 1, ".") )
continue;
entries.push_back( dirEntry->d_name );
}
closedir(dp);
return Status();
}
//----------------------------------------------------------------------------
// Process a config file and return key-value pairs
//----------------------------------------------------------------------------
Status Utils::ProcessConfig( std::map<std::string, std::string> &config,
const std::string &file )
{
config.clear();
std::ifstream inFile( file.c_str() );
if( !inFile.good() )
return Status( stError, errOSError, errno );
errno = 0;
std::string line;
while( std::getline( inFile, line ) )
{
if( line.empty() || line[0] == '#' )
continue;
std::vector<std::string> elems;
splitString( elems, line, "=" );
if( elems.size() != 2 )
return Status( stError, errConfig );
std::string key = elems[0]; Trim( key );
std::string value = elems[1]; Trim( value );
config[key] = value;
}
if( errno )
return Status( stError, errOSError, errno );
return Status();
}
//------------------------------------------------------------------------
//------------------------------------------------------------------------
Status Utils::ProcessConfigDir( std::map<std::string, std::string> &config,
const std::string &dir )
{
Log *log = DefaultEnv::GetLog();
log->Debug( UtilityMsg, "Processing configuration files in %s...",
dir.c_str());
std::vector<std::string> entries;
Status st = Utils::GetDirectoryEntries( entries, dir );
if( !st.IsOK() )
{
log->Debug( UtilityMsg, "Unable to process directory %s: %s",
dir.c_str(), st.ToString().c_str() );
return st;
}
static const std::string suffix = ".conf";
for( auto &entry : entries )
{
std::string confFile = dir + "/" + entry;
if( confFile.length() <= suffix.length() )
continue;
if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
continue;
st = ProcessConfig( config, confFile );
if( !st.IsOK() )
{
log->Debug( UtilityMsg, "Unable to process configuration file %s: %s",
confFile.c_str(), st.ToString().c_str() );
}
}
return Status();
}
//----------------------------------------------------------------------------
// Trim a string
//----------------------------------------------------------------------------
void Utils::Trim( std::string &str )
{
str.erase( str.begin(),
std::find_if( str.begin(), str.end(), isNotSpace ) );
str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
str.end() );
}
//----------------------------------------------------------------------------
// Log property list
//----------------------------------------------------------------------------
void Utils::LogPropertyList( Log *log,
uint64_t topic,
const char *format,
const PropertyList &list )
{
if( unlikely(log->GetLevel() >= Log::DumpMsg) ) {
PropertyList::PropertyMap::const_iterator it;
std::string keyVals;
for (it = list.begin(); it != list.end(); ++it)
keyVals += "'" + it->first + "' = '" + obfuscateAuth(it->second) + "', ";
keyVals.erase(keyVals.length() - 2, 2);
log->Dump(topic, format, keyVals.c_str());
}
}
//----------------------------------------------------------------------------
// Print a char array as hex
//----------------------------------------------------------------------------
std::string Utils::Char2Hex( uint8_t *array, uint16_t size )
{
char *hex = new char[2*size+1];
for( uint16_t i = 0; i < size; ++i )
snprintf( hex+(2*i), 3, "%02x", (int)array[i] );
std::string result = hex;
delete [] hex;
return result;
}
//----------------------------------------------------------------------------
// Normalize checksum
//----------------------------------------------------------------------------
std::string Utils::NormalizeChecksum( const std::string &name,
const std::string &checksum )
{
if( name == "adler32" || name == "crc32" )
{
size_t i;
for( i = 0; i < checksum.length(); ++i )
if( checksum[i] != '0' )
break;
return checksum.substr(i);
}
return checksum;
}
//----------------------------------------------------------------------------
// Get supported checksum types for given URL
//----------------------------------------------------------------------------
std::vector<std::string> Utils::GetSupportedCheckSums( const XrdCl::URL &url )
{
std::vector<std::string> ret;
FileSystem fs( url );
Buffer arg; arg.FromString( "chksum" );
Buffer *resp = 0;
XRootDStatus st = fs.Query( QueryCode::Config, arg, resp );
if( st.IsOK() )
{
std::string response = resp->ToString();
if( response != "chksum" )
{
// we are expecting a response of format: '0:zcrc32,1:adler32'
std::vector<std::string> result;
Utils::splitString( result, response, "," );
std::vector<std::string>::iterator itr = result.begin();
for( ; itr != result.end(); ++itr )
{
size_t pos = itr->find( ':' );
if( pos == std::string::npos ) continue;
std::string cksname = itr->substr( pos + 1 );
// remove all white spaces
cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
cksname.end() );
ret.push_back( std::move( cksname ) );
}
}
}
return ret;
}
//------------------------------------------------------------------------
//------------------------------------------------------------------------
bool Utils::CheckEC( const Message *req, const URL &url )
{
#ifdef WITH_XRDEC
// make sure that if we will be writing it is a new file
ClientRequest *request = (ClientRequest*)req->GetBuffer();
uint16_t options = ntohs( request->open.options );
bool open_wrt = ( options & kXR_open_updt ) || ( options & kXR_open_wrto );
bool open_new = ( options & kXR_new );
if( open_wrt && !open_new ) return false;
const URL::ParamsMap &params = url.GetParams();
// make sure all the xrdec. tokens are present and the values are sane
URL::ParamsMap::const_iterator itr = params.find( "xrdec.nbdta" );
if( itr == params.end() ) return false;
size_t nbdta = std::stoul( itr->second );
itr = params.find( "xrdec.nbprt" );
if( itr == params.end() ) return false;
size_t nbprt = std::stoul( itr->second );
itr = params.find( "xrdec.blksz" );
if( itr == params.end() ) return false;
itr = params.find( "xrdec.plgr" );
if( itr == params.end() ) return false;
std::vector<std::string> plgr;
splitString( plgr, itr->second, "," );
if( plgr.size() < nbdta + nbprt ) return false;
itr = params.find( "xrdec.objid" );
if( itr == params.end() ) return false;
itr = params.find( "xrdec.format" );
if( itr == params.end() ) return false;
size_t format = std::stoul( itr->second );
if( format != 1 ) return false; // TODO use constant
itr = params.find( "xrdec.dtacgi" );
if( itr != params.end() )
{
std::vector<std::string> dtacgi;
splitString( dtacgi, itr->second, "," );
if( plgr.size() != dtacgi.size() ) return false;
}
itr = params.find( "xrdec.mdtacgi" );
if( itr != params.end() )
{
std::vector<std::string> mdtacgi;
splitString( mdtacgi, itr->second, "," );
if( plgr.size() != mdtacgi.size() ) return false;
}
itr = params.find( "xrdec.cosc" );
if( itr == params.end() ) return false;
std::string cosc = itr->second;
if( cosc != "true" && cosc != "false" ) return false;
return true;
#else
return false;
#endif
}
//----------------------------------------------------------------------------
//----------------------------------------------------------------------------
std::string Utils::InferChecksumType( const XrdCl::URL &source,
const XrdCl::URL &destination,
bool zip)
{
//--------------------------------------------------------------------------
// If both files are local we won't be checksumming at all
//--------------------------------------------------------------------------
if( source.IsLocalFile() && !source.IsMetalink() && destination.IsLocalFile() ) return std::string();
// checksums supported by local files
std::set<std::string> local_supported;
local_supported.insert( "adler32" );
local_supported.insert( "crc32" );
local_supported.insert( "md5" );
local_supported.insert( "zcrc32" );
std::vector<std::string> srccks;
if( source.IsMetalink() )
{
int useMtlnCksum = DefaultZipMtlnCksum;
Env *env = DefaultEnv::GetEnv();
env->GetInt( "ZipMtlnCksum", useMtlnCksum );
//------------------------------------------------------------------------
// In case of ZIP use other checksums than zcrc32 only if the user
// requested it explicitly.
//------------------------------------------------------------------------
if( !zip || ( zip && useMtlnCksum ) )
{
RedirectorRegistry &registry = RedirectorRegistry::Instance();
VirtualRedirector *redirector = registry.Get( source );
std::vector<std::string> cks = redirector->GetSupportedCheckSums();
srccks.insert( srccks.end(), cks.begin(), cks.end() );
}
}
if( zip )
{
//------------------------------------------------------------------------
// In case of ZIP we can always extract the checksum from the archive
//------------------------------------------------------------------------
srccks.push_back( "zcrc32" );
}
else if( source.GetProtocol() == "root" || source.GetProtocol() == "xroot" )
{
//------------------------------------------------------------------------
// If the source is a remote endpoint query the supported checksums
//------------------------------------------------------------------------
std::vector<std::string> cks = GetSupportedCheckSums( source );
srccks.insert( srccks.end(), cks.begin(), cks.end() );
}
std::vector<std::string> dstcks;
if( destination.GetProtocol() == "root" ||
destination.GetProtocol() == "xroot" )
{
//------------------------------------------------------------------------
// If the destination is a remote endpoint query the supported checksums
//------------------------------------------------------------------------
std::vector<std::string> cks = GetSupportedCheckSums( destination );
dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
}
//--------------------------------------------------------------------------
// Now we have all the information we need, we can infer the right checksum
// type!!!
//
// First check if source is local
//--------------------------------------------------------------------------
if( source.IsLocalFile() && !source.IsMetalink() )
{
std::vector<std::string>::iterator itr = dstcks.begin();
for( ; itr != dstcks.end(); ++itr )
if( local_supported.count( *itr ) ) return *itr;
return std::string();
}
//--------------------------------------------------------------------------
// then check if destination is local
//--------------------------------------------------------------------------
if( destination.IsLocalFile() )
{
std::vector<std::string>::iterator itr = srccks.begin();
for( ; itr != srccks.end(); ++itr )
if( local_supported.count( *itr ) ) return *itr;
return std::string();
}
//--------------------------------------------------------------------------
// if both source and destination are remote look for a checksum that can
// satisfy both
//--------------------------------------------------------------------------
std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
std::vector<std::string>::iterator itr = srccks.begin();
for( ; itr != srccks.end(); ++itr )
if( dst_supported.count( *itr ) ) return *itr;
return std::string();
}
//----------------------------------------------------------------------------
//----------------------------------------------------------------------------
void Utils::SplitChunks( std::vector<ChunkList> &listsvec,
const ChunkList &chunks,
const uint32_t maxcs,
const size_t maxc )
{
listsvec.clear();
if( !chunks.size() ) return;
listsvec.emplace_back();
ChunkList *c = &listsvec.back();
const size_t cs = chunks.size();
size_t idx = 0;
size_t nc = 0;
ChunkInfo tmpc;
c->reserve( cs );
while( idx < cs )
{
if( maxc && nc >= maxc )
{
listsvec.emplace_back();
c = &listsvec.back();
c->reserve( cs - idx );
nc = 0;
}
if( tmpc.length == 0 )
tmpc = chunks[idx];
if( maxcs && tmpc.length > maxcs )
{
c->emplace_back( tmpc.offset, maxcs, tmpc.buffer );
tmpc.offset += maxcs;
tmpc.length -= maxcs;
tmpc.buffer = static_cast<char*>( tmpc.buffer ) + maxcs;
}
else
{
c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer );
tmpc.length = 0;
++idx;
}
++nc;
}
}
}
ClientOpenRequest::options
kXR_unt16 options
Definition XProtocol.hh:481
kXR_open_wrto
@ kXR_open_wrto
Definition XProtocol.hh:469
kXR_open_updt
@ kXR_open_updt
Definition XProtocol.hh:457
kXR_new
@ kXR_new
Definition XProtocol.hh:455
ClientRequest::open
struct ClientOpenRequest open
Definition XProtocol.hh:860
ClientRequest
Definition XProtocol.hh:845
XrdClCheckSumManager.hh
XrdClConstants.hh
XrdClDefaultEnv.hh
XrdClFileSystem.hh
XrdClMessage.hh
XrdClOptimizers.hh
unlikely
#define unlikely(x)
Definition XrdClOptimizers.hh:27
XrdClRedirectorRegistry.hh
XrdClUtils.hh
XrdNetAddr.hh
obfuscateAuth
std::string obfuscateAuth(const std::string &input)
Definition XrdOucUtils.cc:1534
opendir
#define opendir(a)
Definition XrdPosix.hh:78
closedir
#define closedir(a)
Definition XrdPosix.hh:50
readdir
#define readdir(a)
Definition XrdPosix.hh:86
opts
struct myOpts opts
XrdCksData
Definition XrdCksData.hh:38
XrdCksData::Set
int Set(const char *csName)
Definition XrdCksData.hh:81
XrdCksData::Get
int Get(char *Buff, int Blen)
Definition XrdCksData.hh:69
XrdCl::Buffer
Binary blob representation.
Definition XrdClBuffer.hh:34
XrdCl::Buffer::FromString
void FromString(const std::string str)
Fill the buffer from a string.
Definition XrdClBuffer.hh:205
XrdCl::Buffer::GetBuffer
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
Definition XrdClBuffer.hh:72
XrdCl::Buffer::ToString
std::string ToString() const
Convert the buffer to a string.
Definition XrdClBuffer.hh:215
XrdCl::CheckSumManager
Manage the checksum calc objects.
Definition XrdClCheckSumManager.hh:42
XrdCl::CheckSumManager::Calculate
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
Definition XrdClCheckSumManager.cc:100
XrdCl::DefaultEnv::GetCheckSumManager
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
Definition XrdClDefaultEnv.cc:642
XrdCl::DefaultEnv::GetLog
static Log * GetLog()
Get default log.
Definition XrdClDefaultEnv.cc:493
XrdCl::DefaultEnv::GetEnv
static Env * GetEnv()
Get default client environment.
Definition XrdClDefaultEnv.cc:445
XrdCl::Env
Definition XrdClEnv.hh:38
XrdCl::Env::GetString
bool GetString(const std::string &key, std::string &value)
Definition XrdClEnv.cc:31
XrdCl::Env::GetInt
bool GetInt(const std::string &key, int &value)
Definition XrdClEnv.cc:89
XrdCl::FileSystem
Send file/filesystem queries to an XRootD cluster.
Definition XrdClFileSystem.hh:203
XrdCl::FileSystem::Query
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Definition XrdClFileSystem.cc:1255
XrdCl::Log
Handle diagnostics.
Definition XrdClLog.hh:101
XrdCl::Log::DumpMsg
@ DumpMsg
print details of the request and responses
Definition XrdClLog.hh:113
XrdCl::Log::Error
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition XrdClLog.cc:231
XrdCl::Log::GetLevel
LogLevel GetLevel() const
Get the log level.
Definition XrdClLog.hh:258
XrdCl::Log::Dump
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
Definition XrdClLog.cc:299
XrdCl::Log::Debug
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition XrdClLog.cc:282
XrdCl::Message
The message representation used throughout the system.
Definition XrdClMessage.hh:32
XrdCl::PropertyList
A key-value pair map storing both keys and values as strings.
Definition XrdClPropertyList.hh:41
XrdCl::PropertyList::end
PropertyMap::const_iterator end() const
Get the end iterator.
Definition XrdClPropertyList.hh:168
XrdCl::PropertyList::begin
PropertyMap::const_iterator begin() const
Get the begin iterator.
Definition XrdClPropertyList.hh:160
XrdCl::RedirectorRegistry
Singleton access to URL to virtual redirector mapping.
Definition XrdClRedirectorRegistry.hh:119
XrdCl::RedirectorRegistry::Instance
static RedirectorRegistry & Instance()
Returns reference to the single instance.
Definition XrdClRedirectorRegistry.cc:29
XrdCl::RedirectorRegistry::Get
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
Definition XrdClRedirectorRegistry.cc:116
XrdCl::URL
URL representation.
Definition XrdClURL.hh:31
XrdCl::URL::GetPath
const std::string & GetPath() const
Get the path.
Definition XrdClURL.hh:217
XrdCl::URL::IsMetalink
bool IsMetalink() const
Is it a URL to a metalink.
Definition XrdClURL.cc:465
XrdCl::URL::ParamsMap
std::map< std::string, std::string > ParamsMap
Definition XrdClURL.hh:33
XrdCl::URL::GetHostName
const std::string & GetHostName() const
Get the name of the target host.
Definition XrdClURL.hh:170
XrdCl::URL::IsLocalFile
bool IsLocalFile() const
Definition XrdClURL.cc:474
XrdCl::URL::GetParams
const ParamsMap & GetParams() const
Get the URL params.
Definition XrdClURL.hh:244
XrdCl::URL::GetProtocol
const std::string & GetProtocol() const
Get the protocol.
Definition XrdClURL.hh:118
XrdCl::URL::GetPort
int GetPort() const
Get the target port.
Definition XrdClURL.hh:188
XrdCl::Utils::TimeToString
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
Definition XrdClUtils.cc:256
XrdCl::Utils::CheckTPCLite
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
Definition XrdClUtils.cc:426
XrdCl::Utils::LogHostAddresses
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
Definition XrdClUtils.cc:234
XrdCl::Utils::NormalizeChecksum
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
Definition XrdClUtils.cc:648
XrdCl::Utils::ProcessConfig
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
Definition XrdClUtils.cc:534
XrdCl::Utils::ProcessConfigDir
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
Definition XrdClUtils.cc:566
XrdCl::Utils::FQDNToCC
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
Definition XrdClUtils.cc:490
XrdCl::Utils::InferChecksumType
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
Definition XrdClUtils.cc:771
XrdCl::Utils::LogPropertyList
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
Definition XrdClUtils.cc:617
XrdCl::Utils::Char2Hex
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
Definition XrdClUtils.cc:635
XrdCl::Utils::splitString
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
Definition XrdClUtils.hh:56
XrdCl::Utils::GetHostAddresses
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
Definition XrdClUtils.cc:140
XrdCl::Utils::GetElapsedMicroSecs
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
Definition XrdClUtils.cc:269
XrdCl::Utils::GetSupportedCheckSums
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
Definition XrdClUtils.cc:665
XrdCl::Utils::String2AddressType
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
Definition XrdClUtils.cc:123
XrdCl::Utils::GetIntParameter
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
Definition XrdClUtils.cc:81
XrdCl::Utils::GetDirectoryEntries
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
Definition XrdClUtils.cc:506
XrdCl::Utils::GetLocalCheckSum
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
Definition XrdClUtils.cc:330
XrdCl::Utils::BytesToString
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
Definition XrdClUtils.cc:367
XrdCl::Utils::Trim
static void Trim(std::string &str)
Trim a string.
Definition XrdClUtils.cc:606
XrdCl::Utils::CheckEC
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
Definition XrdClUtils.cc:703
XrdCl::Utils::GetRemoteCheckSum
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
Definition XrdClUtils.cc:279
XrdCl::Utils::GetStringParameter
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
Definition XrdClUtils.cc:104
XrdCl::Utils::CheckTPC
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
Definition XrdClUtils.cc:382
XrdCl::Utils::AddressType
AddressType
Address type.
Definition XrdClUtils.hh:87
XrdCl::Utils::IPAll
@ IPAll
Definition XrdClUtils.hh:89
XrdCl::Utils::IPv6
@ IPv6
Definition XrdClUtils.hh:90
XrdCl::Utils::IPv4
@ IPv4
Definition XrdClUtils.hh:91
XrdCl::Utils::IPAuto
@ IPAuto
Definition XrdClUtils.hh:88
XrdCl::Utils::IPv4Mapped6
@ IPv4Mapped6
Definition XrdClUtils.hh:92
XrdCl::Utils::SplitChunks
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
Definition XrdClUtils.cc:875
XrdCl::VirtualRedirector
An interface for metadata redirectors.
Definition XrdClRedirectorRegistry.hh:60
XrdCl::VirtualRedirector::GetSupportedCheckSums
virtual std::vector< std::string > GetSupportedCheckSums() const =0
XrdCl::XRootDStatus
Request status.
Definition XrdClXRootDResponses.hh:219
XrdCl::XRootDStatus::GetErrorMessage
const std::string & GetErrorMessage() const
Get error message.
Definition XrdClXRootDResponses.hh:242
XrdCl::XRootDStatus::SetErrorMessage
void SetErrorMessage(const std::string &message)
Set the error message.
Definition XrdClXRootDResponses.hh:250
XrdCl::XRootDStatus::ToStr
std::string ToStr() const
Convert to string.
Definition XrdClXRootDResponses.hh:258
XrdNetAddrInfo::isMapped
bool isMapped() const
Definition XrdNetAddrInfo.hh:155
XrdNetAddrInfo::IPv4
@ IPv4
Definition XrdNetAddrInfo.hh:144
XrdNetAddrInfo::IPv6
@ IPv6
Definition XrdNetAddrInfo.hh:144
XrdNetAddrInfo::isIPType
bool isIPType(IPType ipType) const
Definition XrdNetAddrInfo.hh:146
XrdNetAddrInfo::fmtAdv6
@ fmtAdv6
Definition XrdNetAddrInfo.hh:102
XrdNetUtils::GetAddrs
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
Definition XrdNetUtils.cc:274
XrdNetUtils::AddrOpts
AddrOpts
Definition XrdNetUtils.hh:143
XrdNetUtils::prefAuto
@ prefAuto
Definition XrdNetUtils.hh:145
XrdNetUtils::allV4Map
@ allV4Map
Definition XrdNetUtils.hh:143
XrdNetUtils::order46
@ order46
Definition XrdNetUtils.hh:145
XrdNetUtils::allIPMap
@ allIPMap
Definition XrdNetUtils.hh:143
XrdNetUtils::order64
@ order64
Definition XrdNetUtils.hh:145
XrdNetUtils::onlyIPv4
@ onlyIPv4
Definition XrdNetUtils.hh:144
XrdNetUtils::onlyIPv6
@ onlyIPv6
Definition XrdNetUtils.hh:144
XrdCl
Definition XrdClAction.hh:34
XrdCl::suPartial
const uint16_t suPartial
Definition XrdClStatus.hh:41
XrdCl::errInvalidAddr
const uint16_t errInvalidAddr
Definition XrdClStatus.hh:71
XrdCl::stFatal
const uint16_t stFatal
Fatal error, it's still an error.
Definition XrdClStatus.hh:33
XrdCl::stError
const uint16_t stError
An error occurred that could potentially be retried.
Definition XrdClStatus.hh:32
XrdCl::errInternal
const uint16_t errInternal
Internal error.
Definition XrdClStatus.hh:56
XrdCl::stOK
const uint16_t stOK
Everything went OK.
Definition XrdClStatus.hh:31
XrdCl::DefaultIPNoShuffle
const int DefaultIPNoShuffle
Definition XrdClConstants.hh:94
XrdCl::errConfig
const uint16_t errConfig
System misconfigured.
Definition XrdClStatus.hh:55
XrdCl::errOSError
const uint16_t errOSError
Definition XrdClStatus.hh:61
XrdCl::UtilityMsg
const uint64_t UtilityMsg
Definition XrdClConstants.hh:33
XrdCl::errInvalidResponse
const uint16_t errInvalidResponse
Definition XrdClStatus.hh:99
XrdCl::ChunkList
std::vector< ChunkInfo > ChunkList
List of chunks.
Definition XrdClXRootDResponses.hh:1056
XrdCl::errNotSupported
const uint16_t errNotSupported
Definition XrdClStatus.hh:62
XrdCl::DefaultPreferIPv4
const int DefaultPreferIPv4
Definition XrdClConstants.hh:85
XrdCl::errCheckSumError
const uint16_t errCheckSumError
Definition XrdClStatus.hh:101
XrdCl::value
Response NullRef< Response >::value
Definition XrdClOperationHandlers.hh:128
XrdCl::DefaultZipMtlnCksum
const int DefaultZipMtlnCksum
Definition XrdClConstants.hh:93
XrdCl::ChunkInfo
Describe a data chunk for vector read.
Definition XrdClXRootDResponses.hh:918
XrdCl::ChunkInfo::buffer
void * buffer
length of the chunk
Definition XrdClXRootDResponses.hh:951
XrdCl::ChunkInfo::length
uint32_t length
offset in the file
Definition XrdClXRootDResponses.hh:950
XrdCl::ChunkInfo::offset
uint64_t offset
Definition XrdClXRootDResponses.hh:949
XrdCl::QueryCode::Config
@ Config
Query server configuration.
Definition XrdClFileSystem.hh:54
XrdCl::QueryCode::Checksum
@ Checksum
Query file checksum.
Definition XrdClFileSystem.hh:56
XrdCl::Status
Procedure execution status.
Definition XrdClStatus.hh:115
XrdCl::Status::status
uint16_t status
Status of the execution.
Definition XrdClStatus.hh:146
XrdCl::Status::IsOK
bool IsOK() const
We're fine.
Definition XrdClStatus.hh:124
XrdCl::Status::ToString
std::string ToString() const
Create a string representation.
Definition XrdClStatus.cc:97