libzypp 17.25.7
Fetcher.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
12#include <iostream>
13#include <fstream>
14#include <list>
15#include <map>
16
17#include <zypp/base/Easy.h>
19#include <zypp/base/LogTools.h>
20#include <zypp/base/PtrTypes.h>
22#include <zypp/base/String.h>
23#include <zypp/Fetcher.h>
24#include <zypp/ZYppFactory.h>
25#include <zypp/CheckSum.h>
29
30#undef ZYPP_BASE_LOGGER_LOGGROUP
31#define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
32
34namespace zypp
35{
36
43 {
45 : location(loc)
46 {}
51 };
52
53 typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
54
57 {
58 bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
59 {
60 if ( lhs == rhs )
61 return false; // incl. NULL == NULL
62 if ( ! lhs )
63 return true; // NULL < nonNULL
64 if ( ! rhs )
65 return false; // nonNULL > NULL
66 // both nonNULL ==> compare medianr and path
67 if ( lhs->location.medianr() == rhs->location.medianr() )
68 return lhs->location.filename() < rhs->location.filename();
69 //else
70 return lhs->location.medianr() < rhs->location.medianr();
71 }
72 };
73
79 {
80 enum Flag
81 {
82 None = 0x0000,
83 Directory = 0x0001,
84 Recursive = 0x0002,
86 // check checksums even if there is no such
87 // checksum (warns of no checksum)
89 };
91
92
93 FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname())
94 : location(loc)
95 , deltafile(dfile)
96 , flags(None)
97 {
98 //MIL << location << endl;
99 }
100
102 {
103 //MIL << location << " | * " << checkers.size() << endl;
104 }
105
108 //CompositeFileChecker checkers;
109 std::list<FileChecker> checkers;
110 Flags flags;
111 };
112
114 typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
115
116 std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
117 {
118 return str << obj->location;
119 }
120
122 //
123 // CLASS NAME : Fetcher::Impl
124 //
127 {
128 friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
129
130 public:
131 Impl();
132
133 ~Impl() {}
134
135 void setOptions( Fetcher::Options options );
136 Fetcher::Options options() const;
137
138 void addIndex( const OnMediaLocation &resource );
139
140 void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
141 void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
142
143 void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
144 void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() );
145 void addCachePath( const Pathname &cache_dir );
146 void reset();
147 void start( const Pathname &dest_dir,
148 MediaSetAccess &media,
149 const ProgressData::ReceiverFnc & progress_receiver );
150
152 static shared_ptr<Impl> nullimpl()
153 {
154 static shared_ptr<Impl> _nullimpl( new Impl );
155 return _nullimpl;
156 }
157 private:
161 void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
162
166 void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
167
175 void readIndex( const Pathname &index, const Pathname &basedir );
176
178 void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
179
181 void readContentFileIndex( const Pathname &index, const Pathname &basedir );
182
184 void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
185
191 Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
196 void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
197
201 void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
202 const Pathname &dest_dir, FetcherJob::Flags flags );
203
207 void autoaddIndexes( const filesystem::DirContent &content,
208 MediaSetAccess &media,
209 const OnMediaLocation &resource,
210 const Pathname &dest_dir );
214 void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
215
216 private:
217 friend Impl * rwcowClone<Impl>( const Impl * rhs );
219 Impl * clone() const
220 { return new Impl( *this ); }
221
222 std::list<FetcherJob_Ptr> _resources;
223 std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
224 std::set<Pathname> _caches;
225 // checksums read from the indexes
226 std::map<std::string, CheckSum> _checksums;
227 // cache of dir contents
228 std::map<std::string, filesystem::DirContent> _dircontent;
229
230 Fetcher::Options _options;
231 };
233
234 void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
235 {
236 FetcherJob_Ptr job;
237 job.reset(new FetcherJob(resource, deltafile));
239 _resources.push_back(job);
240 }
241
243 : _options(0)
244 {
245 }
246
247 void Fetcher::Impl::setOptions( Fetcher::Options options )
248 { _options = options; }
249
250 Fetcher::Options Fetcher::Impl::options() const
251 { return _options; }
252
254 bool recursive,
255 const FileChecker &checker )
256 {
257 FetcherJob_Ptr job;
258 job.reset(new FetcherJob(resource));
259 if ( checker )
260 job->checkers.push_back(checker);
261 if ( recursive )
262 job->flags |= FetcherJob::Recursive;
263 job->flags |= FetcherJob::Directory;
264
265 _resources.push_back(job);
266 }
267
269 bool recursive,
270 const FileChecker &checker )
271 {
272 FetcherJob_Ptr job;
273 job.reset(new FetcherJob(resource));
274 if ( checker )
275 job->checkers.push_back(checker);
276 if ( recursive )
277 job->flags |= FetcherJob::Recursive;
278 job->flags |= FetcherJob::Directory;
280
281 _resources.push_back(job);
282
283 }
284
285 void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
286 {
287 FetcherJob_Ptr job;
288 job.reset(new FetcherJob(resource));
289 if ( checker )
290 job->checkers.push_back(checker);
291 _resources.push_back(job);
292 }
293
295 {
296 MIL << "adding index " << resource << endl;
297 _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
298 }
299
300
302 {
303 _resources.clear();
304 _indexes.clear();
305 _checksums.clear();
306 _dircontent.clear();
307 }
308
309 void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
310 {
311 PathInfo info(cache_dir);
312 if ( info.isExist() )
313 {
314 if ( info.isDir() )
315 {
316 DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
317 _caches.insert(cache_dir);
318 }
319 else
320 {
321 // don't add bad cache directory, just log the error
322 ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
323 }
324 }
325 else
326 {
327 ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
328 }
329
330 }
331
332 Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
333 {
334 Pathname ret;
335 // No checksum - no match
336 if ( resource_r.checksum().empty() )
337 return ret;
338
339 // first check in the destination directory
340 Pathname cacheLocation = destDir_r / resource_r.filename();
341 if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
342 {
343 swap( ret, cacheLocation );
344 return ret;
345 }
346
347 MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
348 for( const Pathname & cacheDir : _caches )
349 {
350 cacheLocation = cacheDir / resource_r.filename();
351 if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
352 {
353 MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
354 swap( ret, cacheLocation );
355 return ret;
356 }
357 }
358
359 return ret;
360 }
361
362 void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
363 {
364 try
365 {
366 MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
367
368 for ( const FileChecker & chkfnc : checkers_r )
369 {
370 if ( chkfnc )
371 chkfnc( localfile_r );
372 else
373 ERR << "Invalid checker for '" << localfile_r << "'" << endl;
374 }
375
376 }
377 catch ( const FileCheckException &e )
378 {
379 ZYPP_RETHROW(e);
380 }
381 catch ( const Exception &e )
382 {
383 ZYPP_RETHROW(e);
384 }
385 catch (...)
386 {
387 ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
388 }
389 }
390
392 MediaSetAccess &media,
393 const OnMediaLocation &resource,
394 const Pathname &dest_dir )
395 {
396 auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
397 {
398 if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
399 return false;
400 // add the index of this directory
401 OnMediaLocation indexloc( resource );
402 indexloc.changeFilename( resource.filename() + index_r );
403 addIndex( indexloc );
404 // we need to read it now
405 downloadAndReadIndexList( media, dest_dir );
406 return true;
407 } );
408
409 if ( _options & AutoAddChecksumsIndexes )
410 {
411 fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
412 }
413 if ( _options & AutoAddContentFileIndexes )
414 {
415 fnc_addIfInContent( "content" );
416 }
417 }
418
420 const OnMediaLocation &resource,
421 filesystem::DirContent &content )
422 {
423 if ( _dircontent.find(resource.filename().asString())
424 != _dircontent.end() )
425 {
426 filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
427
428 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
429 }
430 else
431 {
433 media.dirInfo( tofill,
434 resource.filename(),
435 false /* dots */,
436 resource.medianr());
437 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
438 _dircontent[resource.filename().asString()] = tofill;
439 }
440 }
441
443 const OnMediaLocation &resource,
444 const Pathname &dest_dir, FetcherJob::Flags flags )
445 {
446 // first get the content of the directory so we can add
447 // individual transfer jobs
448 MIL << "Adding directory " << resource.filename() << endl;
450 try {
451 getDirectoryContent(media, resource, content);
452 }
453 catch ( media::MediaFileNotFoundException & exception )
454 {
455 ZYPP_CAUGHT( exception );
456 WAR << "Skipping subtree hidden at " << resource.filename() << endl;
457 return;
458 }
459
460 // this method test for the option flags so indexes are added
461 // only if the options are enabled
462 autoaddIndexes(content, media, resource, dest_dir);
463
464 for ( filesystem::DirContent::const_iterator it = content.begin();
465 it != content.end();
466 ++it )
467 {
468 // skip CHECKSUMS* as they were already retrieved
469 if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
470 continue;
471
472 Pathname filename = resource.filename() + it->name;
473
474 switch ( it->type )
475 {
476 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
478 {
479 CheckSum chksm(resource.checksum());
480 if ( _checksums.find(filename.asString()) != _checksums.end() )
481 {
482 // the checksum can be replaced with the one in the index.
483 chksm = _checksums[filename.asString()];
484 //MIL << "resource " << filename << " has checksum in the index file." << endl;
485 }
486 else
487 WAR << "Resource " << filename << " has no checksum in the index either." << endl;
488
490 enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
491 else
492 enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
493 break;
494 }
495 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
496 if ( flags & FetcherJob::Recursive )
497 addDirJobs(media, filename, dest_dir, flags);
498 break;
499 default:
500 // don't provide devices, sockets, etc.
501 break;
502 }
503 }
504 }
505
506 void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
507 {
508 const OnMediaLocation & resource( jobp_r->location );
509
510 try
511 {
512 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
513
514 // get cached file (by checksum) or provide from media
515 Pathname tmpFile = locateInCache( resource, destDir_r );
516 if ( tmpFile.empty() )
517 {
518 MIL << "Not found in cache, retrieving..." << endl;
519 tmpFile = media_r.provideFile( resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, jobp_r->deltafile );
520 releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
521 }
522
523 // The final destination: locateInCache also checks destFullPath!
524 // If we find a cache match (by checksum) at destFullPath, take
525 // care it gets deleted, in case the validation fails.
526 ManagedFile destFullPath( destDir_r / resource.filename() );
527 if ( tmpFile == destFullPath )
528 destFullPath.setDispose( filesystem::unlink );
529
530 // validate the file (throws if not valid)
531 validate( tmpFile, jobp_r->checkers );
532
533 // move it to the final destination
534 if ( tmpFile == destFullPath )
535 destFullPath.resetDispose(); // keep it!
536 else
537 {
538 if ( assert_dir( destFullPath->dirname() ) != 0 )
539 ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
540
541 if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
542 ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
543 }
544 }
545 catch ( Exception & excpt )
546 {
547 if ( resource.optional() )
548 {
549 ZYPP_CAUGHT( excpt );
550 WAR << "optional resource " << resource << " could not be transferred." << endl;
551 return;
552 }
553 else
554 {
555 excpt.remember( "Can't provide " + resource.filename().asString() );
556 ZYPP_RETHROW( excpt );
557 }
558 }
559 }
560
561 // helper class to consume a content file
563 {
565 {
566 setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
567 }
568
569 void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
570 { _repoindex = data_r; }
571
572 parser::susetags::RepoIndex_Ptr _repoindex;
573 };
574
575 // generic function for reading indexes
576 void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
577 {
578 if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
579 readChecksumsIndex(index, basedir);
580 else if ( index.basename() == "content" )
581 readContentFileIndex(index, basedir);
582 else
583 WAR << index << ": index file format not known" << endl;
584 }
585
586 // reads a content file index
587 void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
588 {
589 ContentReaderHelper reader;
590 reader.parse(index);
591 MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
592 for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
593 {
594 // content file entries don't start with /
595 _checksums[(basedir + it->first).asString()] = it->second;
596 }
597 }
598
599 // reads a CHECKSUMS (old SHA1SUMS) file index
600 void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
601 {
602 std::ifstream in( index.c_str() );
603 if ( ! in.fail() )
604 {
605 std::string buffer;
606 while ( getline( in, buffer ) )
607 {
608
609 if ( buffer[0] == '#' )
610 continue; // simple comment
611
612 CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
613 if ( checksum.empty() )
614 continue; // empty line | unknown cheksum format
615
616 if ( buffer.empty() )
617 {
618 WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
619 continue;
620 }
621
622 _checksums[(basedir/buffer).asString()] = checksum;
623 }
624 }
625 else
626 ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
627 }
628
629 void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
630 {
631 MIL << "downloading index " << resource << endl;
632 // create a new fetcher with a different state to transfer the
633 // file containing checksums and its signature
634 Fetcher fetcher;
635 // signature checker for index. We havent got the signature from
636 // the nextwork yet.
637 SignatureFileChecker sigchecker;
638
639 // build the name of the index and the signature
640 OnMediaLocation idxloc(resource);
641 OnMediaLocation sigloc(resource);
642 OnMediaLocation keyloc(resource);
643
644 // we should not fail the download if those don't exists
645 // the checking will warn later
646 sigloc.setOptional(true);
647 keyloc.setOptional(true);
648
649 // calculate signature and key name
650 sigloc.changeFilename( sigloc.filename().extend(".asc") );
651 keyloc.changeFilename( keyloc.filename().extend(".key") );
652
653 //assert_dir(dest_dir + idxloc.filename().dirname());
654
655 // transfer the signature
656 fetcher.enqueue(sigloc);
657 fetcher.start( dest_dir, media );
658 // if we get the signature, update the checker
659 if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
660 sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
661
662 fetcher.reset();
663
664 // now the key
665 fetcher.enqueue(keyloc);
666 fetcher.start( dest_dir, media );
667 fetcher.reset();
668
669 // try to import the key
670 if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
671 getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
672 else
673 WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
674
675 // now the index itself
676 fetcher.enqueue( idxloc, FileChecker(sigchecker) );
677 fetcher.start( dest_dir, media );
678 fetcher.reset();
679 }
680
681 // this method takes all the user pointed indexes, gets them and also tries to
682 // download their signature, and verify them. After that, its parses each one
683 // to fill the checksum cache.
685 {
686 // if there is no indexes, then just return to avoid
687 // the directory listing
688 if ( _indexes.empty() )
689 {
690 MIL << "No indexes to read." << endl;
691 return;
692 }
693
694 for_( it_idx, _indexes.begin(), _indexes.end() )
695 {
696 if ( (*it_idx)->read )
697 {
698 DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
699 }
700 else
701 {
702 // base::LogControl::TmpLineWriter shutUp;
703 downloadIndex( media, (*it_idx)->location, dest_dir );
704 // now we have the indexes in dest_dir
705 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
706 // Take care we don't process it again
707 MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708 (*it_idx)->read = true;
709 }
710 }
711 MIL << "done reading indexes" << endl;
712 }
713
714 // start processing all fetcher jobs.
715 // it processes any user pointed index first
716 void Fetcher::Impl::start( const Pathname &dest_dir,
717 MediaSetAccess &media,
718 const ProgressData::ReceiverFnc & progress_receiver )
719 {
720 ProgressData progress(_resources.size());
721 progress.sendTo(progress_receiver);
722
723 downloadAndReadIndexList(media, dest_dir);
724
725 for ( const FetcherJob_Ptr & jobp : _resources )
726 {
727 if ( jobp->flags & FetcherJob::Directory )
728 {
729 const OnMediaLocation location(jobp->location);
730 addDirJobs(media, location, dest_dir, jobp->flags);
731 continue;
732 }
733
734 // may be this code can be factored out
735 // together with the autodiscovery of indexes
736 // of addDirJobs
737 if ( ( _options & AutoAddChecksumsIndexes ) ||
738 ( _options & AutoAddContentFileIndexes ) )
739 {
740 // if auto indexing is enabled, then we need to read the
741 // index for each file. We look only in the directory
742 // where the file is. this is expensive of course.
744 getDirectoryContent(media, jobp->location.filename().dirname(), content);
745 // this method test for the option flags so indexes are added
746 // only if the options are enabled
747 MIL << "Autodiscovering signed indexes on '"
748 << jobp->location.filename().dirname() << "' for '"
749 << jobp->location.filename() << "'" << endl;
750
751 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
752
753 // also look in the root of the media
754 content.clear();
755 getDirectoryContent(media, Pathname("/"), content);
756 // this method test for the option flags so indexes are added
757 // only if the options are enabled
758 MIL << "Autodiscovering signed indexes on '"
759 << "/" << "' for '"
760 << jobp->location.filename() << "'" << endl;
761
762 autoaddIndexes(content, media, Pathname("/"), dest_dir);
763 }
764
765 // if the checksum is empty, but the checksum is in one of the
766 // indexes checksum, then add a checker
767 if ( jobp->location.checksum().empty() )
768 {
769 if ( _checksums.find(jobp->location.filename().asString())
770 != _checksums.end() )
771 {
772 CheckSum chksm = _checksums[jobp->location.filename().asString()];
773 ChecksumFileChecker digest_check(chksm);
774 jobp->checkers.push_back(digest_check);
775 }
776 else
777 {
778 // if the index checksum is empty too, we only add the checker
779 // if the AlwaysVerifyChecksum option is set on
780 if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
781 {
782 // add the checker with the empty checksum
783 ChecksumFileChecker digest_check(jobp->location.checksum());
784 jobp->checkers.push_back(digest_check);
785 }
786 }
787 }
788 else
789 {
790 // checksum is not empty, so add a checksum checker
791 ChecksumFileChecker digest_check(jobp->location.checksum());
792 jobp->checkers.push_back(digest_check);
793 }
794
795 // Provide and validate the file. If the file was not transferred
796 // and no exception was thrown, it was an optional file.
797 provideToDest( media, dest_dir, jobp );
798
799 if ( ! progress.incr() )
800 ZYPP_THROW(AbortRequestException());
801 } // for each job
802 }
803
805 inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
806 {
807 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
808 {
809 str << *it_res;
810 }
811 return str;
812 }
813
815 : _pimpl( new Impl() )
816 {}
817
819 {}
820
821 void Fetcher::setOptions( Fetcher::Options options )
822 {
824 }
825
826 Fetcher::Options Fetcher::options() const
827 {
828 return _pimpl->options();
829 }
830
831 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
832 {
833 _pimpl->enqueueDigested(resource, checker, deltafile);
834 }
835
837 bool recursive,
838 const FileChecker &checker )
839 {
840 _pimpl->enqueueDir(resource, recursive, checker);
841 }
842
844 bool recursive,
845 const FileChecker &checker )
846 {
847 _pimpl->enqueueDigestedDir(resource, recursive, checker);
848 }
849
850
851 void Fetcher::addIndex( const OnMediaLocation &resource )
852 {
853 _pimpl->addIndex(resource);
854 }
855
856
857 void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
858 {
859 _pimpl->enqueue(resource, checker);
860 }
861
862 void Fetcher::addCachePath( const Pathname &cache_dir )
863 {
864 _pimpl->addCachePath(cache_dir);
865 }
866
868 {
869 _pimpl->reset();
870 }
871
872 void Fetcher::start( const Pathname &dest_dir,
873 MediaSetAccess &media,
874 const ProgressData::ReceiverFnc & progress_receiver )
875 {
876 _pimpl->start(dest_dir, media, progress_receiver);
877 }
878
879 std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
880 {
881 return str << *obj._pimpl;
882 }
883
885} // namespace zypp
887
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:396
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
#define DBG
Definition: Logger.h:78
#define MIL
Definition: Logger.h:79
#define ERR
Definition: Logger.h:81
#define WAR
Definition: Logger.h:80
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:93
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:162
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:158
bool empty() const
Definition: CheckSum.cc:173
Built in file checkers.
Definition: FileChecker.h:71
Base class for Exception.
Definition: Exception.h:146
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:105
Fetcher implementation.
Definition: Fetcher.cc:127
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:391
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:629
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:587
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:576
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:223
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:226
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:684
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:442
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:219
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:309
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:222
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:600
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:253
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:294
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:228
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:506
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:285
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Definition: Fetcher.cc:234
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
Definition: Fetcher.cc:332
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
Definition: Fetcher.cc:716
std::set< Pathname > _caches
Definition: Fetcher.cc:224
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:268
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:362
Fetcher::Options options() const
Definition: Fetcher.cc:250
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:419
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:152
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:247
Fetcher::Options _options
Definition: Fetcher.cc:230
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:106
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:325
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:851
Options options() const
Get current options.
Definition: Fetcher.cc:826
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:867
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
Definition: Fetcher.cc:872
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:857
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:836
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:821
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
Fetcher()
Default ctor.
Definition: Fetcher.cc:814
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:843
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:862
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:831
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:818
Media access layer responsible for handling files distributed on a set of media with media change and...
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT, const Pathname &deltafile=Pathname())
Provides a file from a media location.
@ PROVIDE_DEFAULT
The user is not asked anything, and the error exception is just propagated.
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
Describes a resource file located on a medium.
bool optional() const
Whether this is an optional resource.
const Pathname & filename() const
The path to the resource on the medium.
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
unsigned medianr() const
The media number the resource is located on.
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
const CheckSum & checksum() const
The checksum of the resource on the server.
OnMediaLocation & setOptional(bool val)
Set whether the resource is optional.
Maintain [min,max] and counter (value) for progress counting.
Definition: ProgressData.h:131
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: ProgressData.h:226
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Definition: ProgressData.h:261
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:276
Checks for the validity of a signature.
Definition: FileChecker.h:94
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:281
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:170
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
String related utilities and Regular expression matching.
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1013
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:662
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1025
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:320
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:845
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:50
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:263
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1023
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
std::ostream & operator<<(std::ostream &str, const Exception &obj)
Definition: Exception.cc:147
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:53
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:114
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:37
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:572
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:569
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:43
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:48
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:44
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:50
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:79
std::list< FileChecker > checkers
Definition: Fetcher.cc:109
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
Definition: Fetcher.cc:93
@ AlwaysVerifyChecksum
Definition: Fetcher.cc:88
Pathname deltafile
Definition: Fetcher.cc:107
OnMediaLocation location
Definition: Fetcher.cc:106
ZYPP_DECLARE_FLAGS(Flags, Flag)
std::set ordering (less semantic)
Definition: Fetcher.cc:57
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Definition: Fetcher.cc:58
Listentry returned by readdir.
Definition: PathInfo.h:532