acquire.h
Go to the documentation of this file.
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
/* ######################################################################
Acquire - File Acquiration
This module contains the Acquire system. It is responsible for bringing
files into the local pathname space. It deals with URIs for files and
URI handlers responsible for downloading or finding the URIs.
Each file to download is represented by an Acquire::Item class subclassed
into a specialization. The Item class can add itself to several URI
acquire queues each prioritized by the download scheduler. When the
system is run the proper URI handlers are spawned and the acquire
queues are fed into the handlers by the schedular until the queues are
empty. This allows for an Item to be downloaded from an alternate source
if the first try turns out to fail. It also allows concurrent downloading
of multiple items from multiple sources as well as dynamic balancing
of load between the sources.
Scheduling of downloads is done on a first ask first get basis. This
preserves the order of the download as much as possible. And means the
fastest source will tend to process the largest number of files.
Internal methods and queues for performing gzip decompression,
md5sum hashing and file copying are provided to allow items to apply
a number of transformations to the data files they are working with.
##################################################################### */
/*}}}*/
/*}}}*/
#ifndef PKGLIB_ACQUIRE_H
#define PKGLIB_ACQUIRE_H
#include <apt-pkg/hashes.h>
#include <apt-pkg/macros.h>
#include <apt-pkg/weakptr.h>
#include <chrono>
#include <string>
#include <vector>
#include <cstddef>
#include <sys/select.h>
#include <sys/time.h>
class pkgAcquireStatus;
class metaIndex;
class APT_PUBLIC pkgAcquire
{
private:
using clock = std::chrono::steady_clock;
using time_point = std::chrono::time_point<clock>;
int LockFD;
void * const d;
public:
class Item;
class Queue;
class Worker;
struct MethodConfig;
struct ItemDesc;
friend class Item;
friend class pkgAcqMetaBase;
friend class Queue;
typedef std::vector<Item *>::iterator ItemIterator;
typedef std::vector<Item *>::const_iterator ItemCIterator;
protected:
std::vector<Item *> Items;
Queue *Queues;
Worker *Workers;
MethodConfig *Configs;
pkgAcquireStatus *Log;
unsigned long ToFetch;
// Configurable parameters for the scheduler
enum QueueStrategy {
QueueHost,
QueueAccess} QueueMode;
bool const Debug;
bool Running;
void Add(Item *Item);
void Remove(Item *Item);
void Add(Worker *Work);
void Remove(Worker *Work);
void Enqueue(ItemDesc &Item);
void Dequeue(Item *Item);
std::string QueueName(std::string URI,MethodConfig const *&Config);
virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet);
virtual bool RunFds(fd_set *RSet,fd_set *WSet);
void Bump();
public:
MethodConfig *GetConfig(std::string Access);
enum RunResult {
Continue,
Failed,
Cancelled};
RunResult Run(int PulseInterval=500000);
void Shutdown();
inline Worker *WorkersBegin() {return Workers;};
Worker *WorkerStep(Worker *I) APT_PURE;
inline ItemIterator ItemsBegin() {return Items.begin();};
inline ItemCIterator ItemsBegin() const {return Items.begin();};
inline ItemIterator ItemsEnd() {return Items.end();};
inline ItemCIterator ItemsEnd() const {return Items.end();};
// Iterate over queued Item URIs
class UriIterator;
UriIterator UriBegin();
UriIterator UriEnd();
bool Clean(std::string Dir);
bool CleanLists(std::string const &Dir);
unsigned long long TotalNeeded();
unsigned long long FetchNeeded();
unsigned long long PartialPresent();
void SetLog(pkgAcquireStatus *Progress) { Log = Progress; }
bool GetLock(std::string const &Lock);
explicit pkgAcquire(pkgAcquireStatus *Log);
pkgAcquire();
virtual ~pkgAcquire();
APT_HIDDEN static std::string URIEncode(std::string const &part);
private:
APT_HIDDEN void Initialize();
};
struct APT_PUBLIC pkgAcquire::ItemDesc : public WeakPointable
{
std::string URI;
std::string Description;
std::string ShortDesc;
Item *Owner;
};
/*}}}*/
class APT_PUBLIC pkgAcquire::Queue
{
friend class pkgAcquire;
friend class pkgAcquire::UriIterator;
friend class pkgAcquire::Worker;
void * const d;
Queue *Next;
protected:
struct QItem : public ItemDesc
{
QItem *Next;
pkgAcquire::Worker *Worker;
std::vector<Item*> Owners;
unsigned long long CurrentSize = 0;
unsigned long long TotalSize = 0;
unsigned long long ResumePoint = 0;
typedef std::vector<Item*>::const_iterator owner_iterator;
void operator =(pkgAcquire::ItemDesc const &I)
{
URI = I.URI;
Description = I.Description;
ShortDesc = I.ShortDesc;
Owners.clear();
Owners.push_back(I.Owner);
Owner = I.Owner;
};
HashStringList GetExpectedHashes() const;
unsigned long long GetMaximumSize() const;
void SyncDestinationFiles() const;
std::string Custom600Headers() const;
int APT_HIDDEN GetPriority() const;
time_point APT_HIDDEN GetFetchAfter() const;
};
std::string Name;
QItem *Items;
pkgAcquire::Worker *Workers;
pkgAcquire *Owner;
signed long PipeDepth;
unsigned long MaxPipeDepth;
public:
bool Enqueue(ItemDesc &Item);
bool Dequeue(Item *Owner);
QItem *FindItem(std::string URI,pkgAcquire::Worker *Owner) APT_PURE;
bool ItemStart(QItem *Itm,unsigned long long Size);
bool ItemDone(QItem *Itm);
bool Startup();
bool Shutdown(bool Final);
bool Cycle();
void Bump();
Queue(std::string const &Name,pkgAcquire * const Owner);
virtual ~Queue();
};
/*}}}*/
class APT_PUBLIC pkgAcquire::UriIterator
{
void * const d;
pkgAcquire::Queue *CurQ;
pkgAcquire::Queue::QItem *CurItem;
public:
inline void operator ++() {operator ++(0);};
void operator ++(int)
{
CurItem = CurItem->Next;
while (CurItem == 0 && CurQ != 0)
{
CurItem = CurQ->Items;
CurQ = CurQ->Next;
}
};
inline pkgAcquire::Queue::QItem const *operator ->() const {return CurItem;};
inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;};
inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;};
explicit UriIterator(pkgAcquire::Queue *Q);
virtual ~UriIterator();
};
/*}}}*/
struct APT_PUBLIC pkgAcquire::MethodConfig
{
class Private;
Private *const d;
MethodConfig *Next;
std::string Access;
std::string Version;
bool SingleInstance;
bool Pipeline;
bool SendConfig;
bool LocalOnly;
bool NeedsCleanup;
bool Removable;
MethodConfig();
APT_HIDDEN bool GetAuxRequests() const;
APT_HIDDEN void SetAuxRequests(bool const value);
APT_HIDDEN bool GetSendURIEncoded() const;
APT_HIDDEN void SetSendURIEncoded(bool const value);
virtual ~MethodConfig();
};
/*}}}*/
class APT_PUBLIC pkgAcquireStatus
{
void * const d;
protected:
struct timeval Time;
struct timeval StartTime;
unsigned long long LastBytes;
unsigned long long CurrentCPS;
unsigned long long CurrentBytes;
unsigned long long TotalBytes;
unsigned long long FetchedBytes;
unsigned long long ElapsedTime;
unsigned long TotalItems;
unsigned long CurrentItems;
double Percent;
public:
bool Update;
bool MorePulses;
virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint);
virtual bool MediaChange(std::string Media,std::string Drive) = 0;
struct ReleaseInfoChange
{
std::string Type;
std::string From;
std::string To;
std::string Message;
bool DefaultAction;
};
virtual bool ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes);
APT_HIDDEN static bool ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes);
virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {};
virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {};
virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {};
virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {};
virtual bool Pulse(pkgAcquire *Owner);
virtual void Start();
virtual void Stop();
pkgAcquireStatus();
virtual ~pkgAcquireStatus();
};
/*}}}*/
#endif
HashStringList
Definition hashes.h:79
URI
Definition strutl.h:221
WeakPointable
Definition weakptr.h:34
metaIndex
Definition metaindex.h:21
pkgAcquireStatus
A monitor object for downloads controlled by the pkgAcquire class. {{{.
Definition acquire.h:711
pkgAcquireStatus::IMSHit
virtual void IMSHit(pkgAcquire::ItemDesc &)
Invoked when an item is confirmed to be up-to-date.
Definition acquire.h:844
pkgAcquireStatus::ElapsedTime
unsigned long long ElapsedTime
The amount of time that has elapsed since the download started.
Definition acquire.h:753
pkgAcquireStatus::StartTime
struct timeval StartTime
The time at which the download started.
Definition acquire.h:721
pkgAcquireStatus::Percent
double Percent
The estimated percentage of the download (0-100).
Definition acquire.h:767
pkgAcquireStatus::Fail
virtual void Fail(pkgAcquire::ItemDesc &)
Invoked when the process of fetching an item encounters a fatal error.
Definition acquire.h:855
pkgAcquireStatus::Done
virtual void Done(pkgAcquire::ItemDesc &)
Invoked when an item is successfully and completely fetched.
Definition acquire.h:850
pkgAcquireStatus::MediaChange
virtual bool MediaChange(std::string Media, std::string Drive)=0
Invoked when the user should be prompted to change the inserted removable media.
pkgAcquireStatus::pkgAcquireStatus
pkgAcquireStatus()
Initialize all counters to 0 and the time to the current time.
Definition acquire.cc:1374
pkgAcquireStatus::Fetch
virtual void Fetch(pkgAcquire::ItemDesc &)
Invoked when some of an item's data is fetched.
Definition acquire.h:847
pkgAcquireStatus::FetchedBytes
unsigned long long FetchedBytes
The total number of bytes accounted for by items that were successfully fetched.
Definition acquire.h:748
pkgAcquireStatus::Update
bool Update
If true, the download scheduler should call Pulse() at the next available opportunity.
Definition acquire.h:774
pkgAcquireStatus::Time
struct timeval Time
The last time at which this monitor object was updated.
Definition acquire.h:718
pkgAcquireStatus::CurrentBytes
unsigned long long CurrentBytes
The number of bytes fetched as of the most recent call to pkgAcquireStatus::Pulse,...
Definition acquire.h:736
pkgAcquireStatus::MorePulses
bool MorePulses
If true, extra Pulse() invocations will be performed.
Definition acquire.h:782
pkgAcquireStatus::CurrentItems
unsigned long CurrentItems
The number of items that have been successfully downloaded.
Definition acquire.h:763
pkgAcquireStatus::TotalBytes
unsigned long long TotalBytes
The total number of bytes that need to be fetched.
Definition acquire.h:743
pkgAcquireStatus::CurrentCPS
unsigned long long CurrentCPS
The current rate of download as of the most recent call to pkgAcquireStatus::Pulse,...
Definition acquire.h:731
pkgAcquireStatus::LastBytes
unsigned long long LastBytes
The number of bytes fetched as of the previous call to pkgAcquireStatus::Pulse, including local items...
Definition acquire.h:726
pkgAcquireStatus::Fetched
virtual void Fetched(unsigned long long Size, unsigned long long ResumePoint)
Invoked when a local or remote file has been completely fetched.
Definition acquire.cc:1560
pkgAcquireStatus::TotalItems
unsigned long TotalItems
The total number of items that need to be fetched.
Definition acquire.h:760
pkgAcquireStatus::ReleaseInfoChanges
virtual bool ReleaseInfoChanges(metaIndex const *const LastRelease, metaIndex const *const CurrentRelease, std::vector< ReleaseInfoChange > &&Changes)
ask the user for confirmation of changes to infos about a repository
Definition acquire.cc:1565
pkgAcquire::Item
Represents the process by which a pkgAcquire object should retrieve a file or a collection of files.
Definition acquire-item.h:59
pkgAcquire::MethodConfig::Private
Definition acquire.cc:958
pkgAcquire::Queue
A single download queue in a pkgAcquire object. {{{.
Definition acquire.h:407
pkgAcquire::Queue::Shutdown
bool Shutdown(bool Final)
Shut down the worker process associated with this queue.
Definition acquire.cc:1144
pkgAcquire::Queue::Enqueue
bool Enqueue(ItemDesc &Item)
Insert the given fetch request into this queue.
Definition acquire.cc:1016
pkgAcquire::Queue::Items
QItem * Items
The head of the list of items contained in this queue.
Definition acquire.h:485
pkgAcquire::Queue::Cycle
bool Cycle()
Send idle items to the worker process.
Definition acquire.cc:1219
pkgAcquire::Queue::ItemDone
bool ItemDone(QItem *Itm)
Remove the given item from this queue and set its state to pkgAcquire::Item::StatDone.
Definition acquire.cc:1195
pkgAcquire::Queue::ItemStart
bool ItemStart(QItem *Itm, unsigned long long Size)
pkgAcquire::Queue::Owner
pkgAcquire * Owner
the download scheduler with which this queue is associated.
Definition acquire.h:499
pkgAcquire::Queue::Dequeue
bool Dequeue(Item *Owner)
Remove all fetch requests for the given item from this queue.
Definition acquire.cc:1075
pkgAcquire::Queue::PipeDepth
signed long PipeDepth
The number of entries in this queue that are currently being downloaded.
Definition acquire.h:504
pkgAcquire::Queue::Queue
Queue(std::string const &Name, pkgAcquire *const Owner)
Create a new Queue.
Definition acquire.cc:993
pkgAcquire::Queue::Startup
bool Startup()
Start the worker process associated with this queue.
Definition acquire.cc:1104
pkgAcquire::Queue::Bump
void Bump()
Check for items that could be enqueued.
Definition acquire.cc:1267
pkgAcquire::Queue::FindItem
QItem * FindItem(std::string URI, pkgAcquire::Worker *Owner) APT_PURE
Locate an item in this queue.
Definition acquire.cc:1167
pkgAcquire::Queue::Name
std::string Name
The name of this queue.
Definition acquire.h:479
pkgAcquire::Queue::Workers
pkgAcquire::Worker * Workers
The head of the list of workers associated with this queue.
Definition acquire.h:496
pkgAcquire::Queue::MaxPipeDepth
unsigned long MaxPipeDepth
The maximum number of entries that this queue will attempt to download at once.
Definition acquire.h:509
pkgAcquire::UriIterator
Iterates over all the URIs being fetched by a pkgAcquire object. {{{.
Definition acquire.h:607
pkgAcquire::UriIterator::UriIterator
UriIterator(pkgAcquire::Queue *Q)
Create a new UriIterator.
Definition acquire.cc:1588
pkgAcquire::Worker
A fetch subprocess.
Definition acquire-worker.h:48
pkgAcquire
The core download scheduler. {{{.
Definition acquire.h:94
pkgAcquire::ItemsEnd
ItemIterator ItemsEnd()
Get the end iterator of the list of items.
Definition acquire.h:310
pkgAcquire::Debug
bool const Debug
If true, debugging information will be dumped to std::clog.
Definition acquire.h:174
pkgAcquire::WorkersBegin
Worker * WorkersBegin()
Get the first Worker object.
Definition acquire.h:296
pkgAcquire::QueueName
std::string QueueName(std::string URI, MethodConfig const *&Config)
Determine the fetch method and queue of a URI.
Definition acquire.cc:403
pkgAcquire::SetFds
virtual void SetFds(int &Fd, fd_set *RSet, fd_set *WSet)
Build up the set of file descriptors upon which select() should block.
Definition acquire.cc:537
pkgAcquire::Items
std::vector< Item * > Items
A list of items to download.
Definition acquire.h:126
pkgAcquire::Add
void Add(Item *Item)
Add the given item to the list of items.
Definition acquire.cc:229
pkgAcquire::pkgAcquire
pkgAcquire(pkgAcquireStatus *Log)
Construct a new pkgAcquire.
Definition acquire.cc:80
pkgAcquire::Queues
Queue * Queues
The head of the list of active queues.
Definition acquire.h:133
pkgAcquire::Log
pkgAcquireStatus * Log
The progress indicator for this download.
Definition acquire.h:155
pkgAcquire::Configs
MethodConfig * Configs
The head of the list of acquire method configurations.
Definition acquire.h:152
pkgAcquire::ToFetch
unsigned long ToFetch
The number of files which are to be fetched.
Definition acquire.h:158
pkgAcquire::Bump
void Bump()
Check for idle queues with ready-to-fetch items.
Definition acquire.cc:818
pkgAcquire::RunResult
RunResult
Provides information on how a download terminated.
Definition acquire.h:262
pkgAcquire::Failed
@ Failed
Some files failed to download.
Definition acquire.h:267
pkgAcquire::Continue
@ Continue
All files were fetched successfully.
Definition acquire.h:264
pkgAcquire::Cancelled
@ Cancelled
The download was cancelled by the user (i.e., Log's pkgAcquireStatus::Pulse() method returned false).
Definition acquire.h:272
pkgAcquire::GetConfig
MethodConfig * GetConfig(std::string Access)
Retrieve information about a fetch method by name.
Definition acquire.cc:505
pkgAcquire::Dequeue
void Dequeue(Item *Item)
Remove all fetch requests for this item from all queues.
Definition acquire.cc:377
pkgAcquire::Workers
Worker * Workers
The head of the list of active workers.
Definition acquire.h:140
pkgAcquire::Enqueue
void Enqueue(ItemDesc &Item)
Insert the given fetch request into the appropriate queue.
Definition acquire.cc:324
pkgAcquire::Running
bool Running
If true, a download is currently in progress.
Definition acquire.h:176
pkgAcquire::Remove
void Remove(Item *Item)
Remove the given item from the list of items.
Definition acquire.cc:237
pkgAcquire::QueueStrategy
QueueStrategy
Represents the queuing strategy for remote URIs.
Definition acquire.h:163
pkgAcquire::QueueHost
@ QueueHost
Generate one queue for each protocol/host combination; downloads from multiple hosts can proceed in p...
Definition acquire.h:167
pkgAcquire::QueueAccess
@ QueueAccess
Generate a single queue for each protocol; serialize downloads from multiple hosts.
Definition acquire.h:171
pkgAcquire::ItemsBegin
ItemIterator ItemsBegin()
Get the head of the list of items.
Definition acquire.h:306
pkgAcquire::RunFds
virtual bool RunFds(fd_set *RSet, fd_set *WSet)
Definition acquire.cc:561
pkgAcquireStatus::ReleaseInfoChange
Definition acquire.h:812
pkgAcquireStatus::ReleaseInfoChange::From
std::string From
Definition acquire.h:814
pkgAcquireStatus::ReleaseInfoChange::Type
std::string Type
Definition acquire.h:813
pkgAcquireStatus::ReleaseInfoChange::Message
std::string Message
Definition acquire.h:816
pkgAcquireStatus::ReleaseInfoChange::DefaultAction
bool DefaultAction
Definition acquire.h:817
pkgAcquireStatus::ReleaseInfoChange::To
std::string To
Definition acquire.h:815
pkgAcquire::ItemDesc
Represents a single download source from which an item should be downloaded.
Definition acquire.h:391
pkgAcquire::ItemDesc::Description
std::string Description
description of this item.
Definition acquire.h:395
pkgAcquire::ItemDesc::ShortDesc
std::string ShortDesc
shorter description of this item.
Definition acquire.h:397
pkgAcquire::ItemDesc::Owner
Item * Owner
underlying item which is to be downloaded.
Definition acquire.h:399
pkgAcquire::ItemDesc::URI
std::string URI
URI from which to download this item.
Definition acquire.h:393
pkgAcquire::MethodConfig
Information about the properties of a single acquire method. {{{.
Definition acquire.h:644
pkgAcquire::MethodConfig::Access
std::string Access
The name of this acquire method (e.g., http).
Definition acquire.h:656
pkgAcquire::MethodConfig::Pipeline
bool Pipeline
If true, this method supports pipelined downloading.
Definition acquire.h:667
pkgAcquire::MethodConfig::NeedsCleanup
bool NeedsCleanup
If true, the subprocess has to carry out some cleanup actions before shutting down.
Definition acquire.h:686
pkgAcquire::MethodConfig::MethodConfig
MethodConfig()
Set up the default method parameters.
Definition acquire.cc:963
pkgAcquire::MethodConfig::SingleInstance
bool SingleInstance
If true, only one download queue should be created for this method.
Definition acquire.h:664
pkgAcquire::MethodConfig::Removable
bool Removable
If true, this fetch method acquires files from removable media.
Definition acquire.h:689
pkgAcquire::MethodConfig::LocalOnly
bool LocalOnly
If true, this fetch method does not require network access; all files are to be acquired from the loc...
Definition acquire.h:678
pkgAcquire::MethodConfig::Version
std::string Version
The implementation version of this acquire method.
Definition acquire.h:659
pkgAcquire::MethodConfig::Next
MethodConfig * Next
The next link on the acquire method list.
Definition acquire.h:653
pkgAcquire::MethodConfig::SendConfig
bool SendConfig
If true, the worker process should send the entire APT configuration tree to the fetch subprocess whe...
Definition acquire.h:673
pkgAcquire::MethodConfig::d
Private *const d
dpointer placeholder (for later in case we need it)
Definition acquire.h:647
pkgAcquire::Queue::QItem
A single item placed in this queue.
Definition acquire.h:422
pkgAcquire::Queue::QItem::Next
QItem * Next
The next item in the queue.
Definition acquire.h:424
pkgAcquire::Queue::QItem::Worker
pkgAcquire::Worker * Worker
The worker associated with this item, if any.
Definition acquire.h:426
pkgAcquire::Queue::QItem::ResumePoint
unsigned long long ResumePoint
How much of the file was already downloaded prior to starting this worker.
Definition acquire.h:444
pkgAcquire::Queue::QItem::TotalSize
unsigned long long TotalSize
The total number of bytes to be downloaded. Zero if the total size of the final is unknown.
Definition acquire.h:439
pkgAcquire::Queue::QItem::Owners
std::vector< Item * > Owners
The underlying items interested in the download.
Definition acquire.h:429
pkgAcquire::Queue::QItem::CurrentSize
unsigned long long CurrentSize
How many bytes of the file have been downloaded. Zero if the current progress of the file cannot be d...
Definition acquire.h:434