apt  0.9.14.1+nmu1
acquire.h
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire.h,v 1.29.2.1 2003/12/24 23:09:17 mdz Exp $
4 /* ######################################################################
5 
6  Acquire - File Acquiration
7 
8  This module contians the Acquire system. It is responsible for bringing
9  files into the local pathname space. It deals with URIs for files and
10  URI handlers responsible for downloading or finding the URIs.
11 
12  Each file to download is represented by an Acquire::Item class subclassed
13  into a specialization. The Item class can add itself to several URI
14  acquire queues each prioritized by the download scheduler. When the
15  system is run the proper URI handlers are spawned and the the acquire
16  queues are fed into the handlers by the schedular until the queues are
17  empty. This allows for an Item to be downloaded from an alternate source
18  if the first try turns out to fail. It also alows concurrent downloading
19  of multiple items from multiple sources as well as dynamic balancing
20  of load between the sources.
21 
22  Schedualing of downloads is done on a first ask first get basis. This
23  preserves the order of the download as much as possible. And means the
24  fastest source will tend to process the largest number of files.
25 
26  Internal methods and queues for performing gzip decompression,
27  md5sum hashing and file copying are provided to allow items to apply
28  a number of transformations to the data files they are working with.
29 
30  ##################################################################### */
31  /*}}}*/
32  /*}}}*/
58 
66 #ifndef PKGLIB_ACQUIRE_H
67 #define PKGLIB_ACQUIRE_H
68 
69 #include <apt-pkg/macros.h>
70 #include <apt-pkg/weakptr.h>
71 
72 #include <vector>
73 #include <string>
74 
75 #include <sys/time.h>
76 #include <unistd.h>
77 
78 #ifndef APT_8_CLEANER_HEADERS
79 using std::vector;
80 using std::string;
81 #endif
82 
83 class pkgAcquireStatus;
84 
94 {
95  private:
97  int LockFD;
99  void *d;
100 
101  public:
102 
103  class Item;
104  class Queue;
105  class Worker;
106  struct MethodConfig;
107  struct ItemDesc;
108  friend class Item;
109  friend class Queue;
110 
111  typedef std::vector<Item *>::iterator ItemIterator;
112  typedef std::vector<Item *>::const_iterator ItemCIterator;
113 
114  protected:
115 
121  std::vector<Item *> Items;
122 
129 
136 
148 
151 
153  unsigned long ToFetch;
154 
155  // Configurable parameters for the scheduler
156 
166  QueueAccess} QueueMode;
167 
169  bool const Debug;
171  bool Running;
172 
174  void Add(Item *Item);
175 
177  void Remove(Item *Item);
178 
180  void Add(Worker *Work);
181 
183  void Remove(Worker *Work);
184 
191  void Enqueue(ItemDesc &Item);
192 
194  void Dequeue(Item *Item);
195 
206  std::string QueueName(std::string URI,MethodConfig const *&Config);
207 
222  virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet);
223 
234  virtual void RunFds(fd_set *RSet,fd_set *WSet);
235 
242  void Bump();
243 
244  public:
245 
252  MethodConfig *GetConfig(std::string Access);
253 
255  enum RunResult {
258 
261 
266 
278  RunResult Run(int PulseInterval=500000);
279 
283  void Shutdown();
284 
289  inline Worker *WorkersBegin() {return Workers;};
290 
296  Worker *WorkerStep(Worker *I);
297 
299  inline ItemIterator ItemsBegin() {return Items.begin();};
300 
302  inline ItemIterator ItemsEnd() {return Items.end();};
303 
304  // Iterate over queued Item URIs
305  class UriIterator;
311  UriIterator UriBegin();
313  UriIterator UriEnd();
314 
323  bool Clean(std::string Dir);
324 
328  unsigned long long TotalNeeded();
329 
333  unsigned long long FetchNeeded();
334 
338  unsigned long long PartialPresent();
339 
351  bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = "");
352 
353  void SetLog(pkgAcquireStatus *Progress) { Log = Progress; }
354 
356  pkgAcquire(pkgAcquireStatus *Log) __deprecated;
357  pkgAcquire();
358 
364  virtual ~pkgAcquire();
365 
366 };
367 
374 {
376  std::string URI;
378  std::string Description;
380  std::string ShortDesc;
383 };
384  /*}}}*/
390 {
391  friend class pkgAcquire;
392  friend class pkgAcquire::UriIterator;
393  friend class pkgAcquire::Worker;
394 
396  void *d;
397 
399  Queue *Next;
400 
401  protected:
402 
405  {
410 
415  {
416  URI = I.URI;
418  ShortDesc = I.ShortDesc;
419  Owner = I.Owner;
420  };
421  };
422 
424  std::string Name;
425 
431 
442 
445 
449  signed long PipeDepth;
450 
454  unsigned long MaxPipeDepth;
455 
456  public:
457 
463  bool Enqueue(ItemDesc &Item);
464 
469  bool Dequeue(Item *Owner);
470 
479  QItem *FindItem(std::string URI,pkgAcquire::Worker *Owner);
480 
485  bool ItemStart(QItem *Itm,unsigned long long Size);
486 
497  bool ItemDone(QItem *Itm);
498 
506  bool Startup();
507 
517  bool Shutdown(bool Final);
518 
523  bool Cycle();
524 
535  void Bump();
536 
542  Queue(std::string Name,pkgAcquire *Owner);
543 
547  virtual ~Queue();
548 };
549  /*}}}*/
552 {
554  void *d;
555 
557  pkgAcquire::Queue *CurQ;
559  pkgAcquire::Queue::QItem *CurItem;
560 
561  public:
562 
563  inline void operator ++() {operator ++(0);};
564 
565  void operator ++(int)
566  {
567  CurItem = CurItem->Next;
568  while (CurItem == 0 && CurQ != 0)
569  {
570  CurItem = CurQ->Items;
571  CurQ = CurQ->Next;
572  }
573  };
574 
575  inline pkgAcquire::ItemDesc const *operator ->() const {return CurItem;};
576  inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;};
577  inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;};
578 
583  UriIterator(pkgAcquire::Queue *Q) : CurQ(Q), CurItem(0)
584  {
585  while (CurItem == 0 && CurQ != 0)
586  {
587  CurItem = CurQ->Items;
588  CurQ = CurQ->Next;
589  }
590  }
591  virtual ~UriIterator() {};
592 };
593  /*}}}*/
596 {
598  void *d;
599 
605 
607  std::string Access;
608 
610  std::string Version;
611 
616 
618  bool Pipeline;
619 
625 
629  bool LocalOnly;
630 
638 
640  bool Removable;
641 
647  MethodConfig();
648 
649  /* \brief Destructor, empty currently */
650  virtual ~MethodConfig() {};
651 };
652  /*}}}*/
658 {
660  void *d;
661 
662  protected:
663 
665  struct timeval Time;
666 
668  struct timeval StartTime;
669 
673  unsigned long long LastBytes;
674 
678  unsigned long long CurrentCPS;
679 
683  unsigned long long CurrentBytes;
684 
690  unsigned long long TotalBytes;
691 
695  unsigned long long FetchedBytes;
696 
700  unsigned long long ElapsedTime;
701 
707  unsigned long TotalItems;
708 
710  unsigned long CurrentItems;
711 
712  public:
713 
717  bool Update;
718 
726 
733  virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint);
734 
752  virtual bool MediaChange(std::string Media,std::string Drive) = 0;
753 
759  virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {};
760 
762  virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {};
763 
765  virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {};
766 
770  virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {};
771 
782  virtual bool Pulse(pkgAcquire *Owner);
783 
785  virtual void Start();
786 
788  virtual void Stop();
789 
792  virtual ~pkgAcquireStatus() {};
793 };
794  /*}}}*/
797 #endif
std::string Name
The name of this queue.
Definition: acquire.h:424
Queue * Queues
The head of the list of active queues.
Definition: acquire.h:128
void Dequeue(Item *Item)
Remove all fetch requests for this item from all queues.
Definition: acquire.cc:243
pkgAcquire * Owner
the download scheduler with which this queue is associated.
Definition: acquire.h:444
std::string URI
The URI from which to download this item.
Definition: acquire.h:376
void operator=(pkgAcquire::ItemDesc const &I)
Assign the ItemDesc portion of this QItem from another ItemDesc.
Definition: acquire.h:414
Definition: strutl.h:148
Generate a single queue for each protocol; serialize downloads from multiple hosts.
Definition: acquire.h:166
unsigned long CurrentItems
The number of items that have been successfully downloaded.
Definition: acquire.h:710
Represents a single download source from which an item should be downloaded.
Definition: acquire.h:373
unsigned long long FetchNeeded()
Definition: acquire.cc:534
bool const Debug
If true, debugging information will be dumped to std::clog.
Definition: acquire.h:169
virtual void Stop()
Invoked when the Acquire process stops running.
Definition: acquire.cc:932
bool NeedsCleanup
If true, the subprocess has to carry out some cleanup actions before shutting down.
Definition: acquire.h:637
Queue(std::string Name, pkgAcquire *Owner)
Create a new Queue.
Definition: acquire.cc:587
Some files failed to download.
Definition: acquire.h:260
bool Startup()
Start the worker process associated with this queue.
Definition: acquire.cc:670
bool Pipeline
If true, this method supports pipelined downloading.
Definition: acquire.h:618
std::vector< Item * > Items
A list of items to download.
Definition: acquire.h:121
RunResult
Provides information on how a download terminated.
Definition: acquire.h:255
void Bump()
Check for idle queues with ready-to-fetch items.
Definition: acquire.cc:459
Iterates over all the URIs being fetched by a pkgAcquire object. {{{.
Definition: acquire.h:551
Generate one queue for each protocol/host combination; downloads from multiple hosts can proceed in p...
Definition: acquire.h:162
virtual ~Queue()
Definition: acquire.cc:600
QueueStrategy
Represents the queuing strategy for remote URIs.
Definition: acquire.h:158
bool Clean(std::string Dir)
Definition: acquire.cc:477
All files were fetched successfully.
Definition: acquire.h:257
bool Update
If true, the download scheduler should call Pulse() at the next available opportunity.
Definition: acquire.h:717
Worker * Workers
The head of the list of active workers.
Definition: acquire.h:135
QItem * FindItem(std::string URI, pkgAcquire::Worker *Owner)
Locate an item in this queue.
Definition: acquire.cc:723
unsigned long long CurrentCPS
The current rate of download as of the most recent call to pkgAcquireStatus::Pulse, in bytes per second.
Definition: acquire.h:678
bool LocalOnly
If true, this fetch method does not require network access; all files are to be acquired from the loc...
Definition: acquire.h:629
The core download scheduler. {{{.
Definition: acquire.h:93
virtual void SetFds(int &Fd, fd_set *RSet, fd_set *WSet)
Build up the set of file descriptors upon which select() should block.
Definition: acquire.cc:340
bool Shutdown(bool Final)
Shut down the worker process associated with this queue.
Definition: acquire.cc:700
void Shutdown()
Remove all items from this download process, terminate all download workers, and empty all queues...
Definition: acquire.cc:119
std::string QueueName(std::string URI, MethodConfig const *&Config)
Determine the fetch method and queue of a URI.
Definition: acquire.cc:269
UriIterator(pkgAcquire::Queue *Q)
Create a new UriIterator.
Definition: acquire.h:583
ItemIterator ItemsEnd()
Get the end iterator of the list of items.
Definition: acquire.h:302
void Bump()
Check for items that could be enqueued.
Definition: acquire.cc:790
std::string Access
The name of this acquire method (e.g., http).
Definition: acquire.h:607
virtual void Done(pkgAcquire::ItemDesc &)
Invoked when an item is successfully and completely fetched.
Definition: acquire.h:765
A monitor object for downloads controlled by the pkgAcquire class. {{{.
Definition: acquire.h:657
virtual void Start()
Invoked when the Acquire process starts running.
Definition: acquire.cc:915
bool ItemDone(QItem *Itm)
Remove the given item from this queue and set its state to pkgAcquire::Item::StatDone.
Definition: acquire.cc:736
struct timeval Time
The last time at which this monitor object was updated.
Definition: acquire.h:665
unsigned long long ElapsedTime
The amount of time that has elapsed since the download started.
Definition: acquire.h:700
bool SendConfig
If true, the worker process should send the entire APT configuration tree to the fetch subprocess whe...
Definition: acquire.h:624
virtual void Fail(pkgAcquire::ItemDesc &)
Invoked when the process of fetching an item encounters a fatal error.
Definition: acquire.h:770
bool Dequeue(Item *Owner)
Remove all fetch requests for the given item from this queue.
Definition: acquire.cc:641
unsigned long long PartialPresent()
Definition: acquire.cc:546
void Remove(Item *Item)
Remove the given item from the list of items.
Definition: acquire.cc:148
pkgAcquire::Worker * Worker
The worker associated with this item, if any.
Definition: acquire.h:409
virtual void Fetch(pkgAcquire::ItemDesc &)
Invoked when some of an item&#39;s data is fetched.
Definition: acquire.h:762
Worker * WorkersBegin()
Get the first #Worker object.
Definition: acquire.h:289
virtual void RunFds(fd_set *RSet, fd_set *WSet)
Definition: acquire.cc:364
std::string Version
The implementation version of this acquire method.
Definition: acquire.h:610
pkgAcquire::Worker * Workers
The head of the list of workers associated with this queue.
Definition: acquire.h:441
std::string Description
Definition: acquire.h:378
virtual bool MediaChange(std::string Media, std::string Drive)=0
Invoked when the user should be prompted to change the inserted removable media.
The download was cancelled by the user (i.e., Log&#39;s pkgAcquireStatus::Pulse() method returned false)...
Definition: acquire.h:265
virtual void IMSHit(pkgAcquire::ItemDesc &)
Invoked when an item is confirmed to be up-to-date.
Definition: acquire.h:759
Worker * WorkerStep(Worker *I)
Advance to the next #Worker object.
Definition: acquire.cc:468
void * d
dpointer placeholder (for later in case we need it)
Definition: acquire.h:598
Item * Owner
Definition: acquire.h:382
bool SingleInstance
If true, only one download queue should be created for this method.
Definition: acquire.h:615
pkgAcquireStatus()
Initialize all counters to 0 and the time to the current time.
Definition: acquire.cc:798
unsigned long MaxPipeDepth
The maximum number of entries that this queue will attempt to download at once.
Definition: acquire.h:454
unsigned long ToFetch
The number of files which are to be fetched.
Definition: acquire.h:153
A single item placed in this queue.
Definition: acquire.h:404
QItem * Next
The next item in the queue.
Definition: acquire.h:407
bool Cycle()
Send idle items to the worker process.
Definition: acquire.cc:757
Represents the process by which a pkgAcquire object should {{{ retrieve a file or a collection of fil...
Definition: acquire-item.h:58
ItemIterator ItemsBegin()
Get the head of the list of items.
Definition: acquire.h:299
unsigned long long CurrentBytes
The number of bytes fetched as of the most recent call to pkgAcquireStatus::Pulse, including local items.
Definition: acquire.h:683
MethodConfig * Next
The next link on the acquire method list.
Definition: acquire.h:604
void Add(Item *Item)
Add the given item to the list of items.
Definition: acquire.cc:140
bool Removable
If true, this fetch method acquires files from removable media.
Definition: acquire.h:640
void Enqueue(ItemDesc &Item)
Insert the given fetch request into the appropriate queue.
Definition: acquire.cc:200
virtual void Fetched(unsigned long long Size, unsigned long long ResumePoint)
Invoked when a local or remote file has been completely fetched.
Definition: acquire.cc:953
pkgAcquireStatus * Log
The progress indicator for this download.
Definition: acquire.h:150
virtual ~pkgAcquire()
Destroy this pkgAcquire object.
Definition: acquire.cc:101
MethodConfig()
Set up the default method parameters.
Definition: acquire.cc:574
RunResult Run(int PulseInterval=500000)
Download all the items that have been Add()ed to this download process.
Definition: acquire.cc:380
signed long PipeDepth
The number of entries in this queue that are currently being downloaded.
Definition: acquire.h:449
virtual bool Pulse(pkgAcquire *Owner)
Periodically invoked while the Acquire process is underway.
Definition: acquire.cc:808
bool MorePulses
If true, extra Pulse() invocations will be performed.
Definition: acquire.h:725
UriIterator UriBegin()
Get the head of the list of enqueued item URIs.
Definition: acquire.cc:558
bool ItemStart(QItem *Itm, unsigned long long Size)
UriIterator UriEnd()
Get the end iterator of the list of enqueued item URIs.
Definition: acquire.cc:566
unsigned long TotalItems
The total number of items that need to be fetched.
Definition: acquire.h:707
unsigned long long FetchedBytes
The total number of bytes accounted for by items that were successfully fetched.
Definition: acquire.h:695
Information about the properties of a single acquire method. {{{.
Definition: acquire.h:595
A single download queue in a pkgAcquire object. {{{.
Definition: acquire.h:389
QItem * Items
The head of the list of items contained in this queue.
Definition: acquire.h:430
bool Setup(pkgAcquireStatus *Progress=NULL, std::string const &Lock="")
Delayed constructor.
Definition: acquire.cc:69
unsigned long long TotalBytes
The total number of bytes that need to be fetched.
Definition: acquire.h:690
unsigned long long LastBytes
The number of bytes fetched as of the previous call to pkgAcquireStatus::Pulse, including local items...
Definition: acquire.h:673
Definition: weakptr.h:34
bool Running
If true, a download is currently in progress.
Definition: acquire.h:171
MethodConfig * Configs
The head of the list of acquire method configurations.
Definition: acquire.h:147
A fetch subprocess.
Definition: acquire-worker.h:45
unsigned long long TotalNeeded()
Definition: acquire.cc:523
MethodConfig * GetConfig(std::string Access)
Retrieve information about a fetch method by name.
Definition: acquire.cc:311
bool Enqueue(ItemDesc &Item)
Insert the given fetch request into this queue.
Definition: acquire.cc:615
std::string ShortDesc
Definition: acquire.h:380
struct timeval StartTime
The time at which the download started.
Definition: acquire.h:668