30#undef ZYPP_BASE_LOGGER_LOGGROUP
31#define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
67 if ( lhs->location.medianr() == rhs->location.medianr() )
68 return lhs->location.filename() < rhs->location.filename();
70 return lhs->location.medianr() < rhs->location.medianr();
118 return str << obj->location;
136 Fetcher::Options
options()
const;
154 static shared_ptr<Impl> _nullimpl(
new Impl );
196 void validate(
const Pathname & localfile_r,
const std::list<FileChecker> & checkers_r );
202 const Pathname &dest_dir, FetcherJob::Flags flags );
217 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
220 {
return new Impl( *
this ); }
223 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
237 job.reset(
new FetcherJob(resource, deltafile));
260 job->checkers.push_back(checker);
265 _resources.push_back(job);
275 job->checkers.push_back(checker);
281 _resources.push_back(job);
290 job->checkers.push_back(checker);
291 _resources.push_back(job);
296 MIL <<
"adding index " << resource << endl;
316 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
317 _caches.insert(cache_dir);
322 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
327 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
343 swap( ret, cacheLocation );
347 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
348 for(
const Pathname & cacheDir : _caches )
350 cacheLocation = cacheDir / resource_r.
filename();
353 MIL <<
"file " << resource_r.
filename() <<
" found in cache " << cacheDir << endl;
354 swap( ret, cacheLocation );
366 MIL <<
"Checking job [" << localfile_r <<
"] (" << checkers_r.size() <<
" checkers )" << endl;
371 chkfnc( localfile_r );
373 ERR <<
"Invalid checker for '" << localfile_r <<
"'" << endl;
396 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool
405 downloadAndReadIndexList( media, dest_dir );
411 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
415 fnc_addIfInContent(
"content" );
424 != _dircontent.end() )
428 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
437 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
444 const Pathname &dest_dir, FetcherJob::Flags flags )
448 MIL <<
"Adding directory " << resource.
filename() << endl;
451 getDirectoryContent(media, resource, content);
456 WAR <<
"Skipping subtree hidden at " << resource.
filename() << endl;
462 autoaddIndexes(content, media, resource, dest_dir);
464 for ( filesystem::DirContent::const_iterator it = content.begin();
480 if ( _checksums.find(filename.
asString()) != _checksums.end() )
483 chksm = _checksums[filename.
asString()];
487 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
497 addDirJobs(media, filename, dest_dir, flags);
512 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard;
515 Pathname tmpFile = locateInCache( resource, destDir_r );
516 if ( tmpFile.
empty() )
518 MIL <<
"Not found in cache, retrieving..." << endl;
527 if ( tmpFile == destFullPath )
531 validate( tmpFile, jobp_r->checkers );
534 if ( tmpFile == destFullPath )
538 if (
assert_dir( destFullPath->dirname() ) != 0 )
550 WAR <<
"optional resource " << resource <<
" could not be transferred." << endl;
570 { _repoindex = data_r; }
579 readChecksumsIndex(index, basedir);
580 else if ( index.
basename() ==
"content" )
581 readContentFileIndex(index, basedir);
583 WAR << index <<
": index file format not known" << endl;
591 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
595 _checksums[(basedir + it->first).
asString()] = it->second;
602 std::ifstream in( index.
c_str() );
606 while (
getline( in, buffer ) )
609 if ( buffer[0] ==
'#' )
616 if ( buffer.empty() )
618 WAR <<
"Missing filename in CHECKSUMS file: " << index.
asString() <<
" (" <<
checksum <<
")" << endl;
631 MIL <<
"downloading index " << resource << endl;
657 fetcher.
start( dest_dir, media );
666 fetcher.
start( dest_dir, media );
673 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
677 fetcher.
start( dest_dir, media );
688 if ( _indexes.empty() )
690 MIL <<
"No indexes to read." << endl;
694 for_( it_idx, _indexes.begin(), _indexes.end() )
696 if ( (*it_idx)->read )
698 DBG <<
"Already read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
703 downloadIndex( media, (*it_idx)->location, dest_dir );
705 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().
dirname() );
707 MIL <<
"Remember read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708 (*it_idx)->read =
true;
711 MIL <<
"done reading indexes" << endl;
721 progress.
sendTo(progress_receiver);
723 downloadAndReadIndexList(media, dest_dir);
730 addDirJobs(media, location, dest_dir, jobp->flags);
744 getDirectoryContent(media, jobp->location.filename().dirname(), content);
747 MIL <<
"Autodiscovering signed indexes on '"
748 << jobp->location.filename().dirname() <<
"' for '"
749 << jobp->location.filename() <<
"'" << endl;
751 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
755 getDirectoryContent(media,
Pathname(
"/"), content);
758 MIL <<
"Autodiscovering signed indexes on '"
760 << jobp->location.filename() <<
"'" << endl;
762 autoaddIndexes(content, media,
Pathname(
"/"), dest_dir);
767 if ( jobp->location.checksum().empty() )
769 if ( _checksums.find(jobp->location.filename().asString())
770 != _checksums.end() )
772 CheckSum chksm = _checksums[jobp->location.filename().asString()];
774 jobp->checkers.push_back(digest_check);
784 jobp->checkers.push_back(digest_check);
792 jobp->checkers.push_back(digest_check);
797 provideToDest( media, dest_dir, jobp );
799 if ( ! progress.
incr() )
807 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
void resetDispose()
Set no dispose function.
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Base class for Exception.
void remember(const Exception &old_r)
Store an other Exception as history.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
std::map< std::string, CheckSum > _checksums
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Impl * clone() const
clone for RWCOW_pointer
void addCachePath(const Pathname &cache_dir)
std::list< FetcherJob_Ptr > _resources
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
void addIndex(const OnMediaLocation &resource)
std::map< std::string, filesystem::DirContent > _dircontent
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
std::set< Pathname > _caches
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Fetcher::Options options() const
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
static shared_ptr< Impl > nullimpl()
Offer default Impl.
void setOptions(Fetcher::Options options)
Fetcher::Options _options
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Options options() const
Get current options.
void reset()
Reset the transfer (jobs) list.
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
void setOptions(Options options)
Set the Fetcher options.
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Maintain [min,max] and counter (value) for progress counting.
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Checks for the validity of a signature.
Wrapper class for ::stat/::lstat.
bool isExist() const
Return whether valid stat info exists.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Pathname dirname() const
Return all but the last component od this path.
const char * c_str() const
String representation.
const std::string & asString() const
String representation.
std::string basename() const
Return the last component of this path.
bool empty() const
Test for an empty path.
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
String related utilities and Regular expression matching.
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
int unlink(const Pathname &path)
Like 'unlink'.
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
std::list< DirEntry > DirContent
Returned by readdir.
std::string getline(std::istream &str)
Read one line from stream.
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
std::string stripFirstWord(std::string &line, const bool ltrim_first)
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Easy-to use interface to the ZYPP dependency resolver.
std::ostream & operator<<(std::ostream &str, const Exception &obj)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
shared_ptr< FetcherJob > FetcherJob_Ptr
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
parser::susetags::RepoIndex_Ptr _repoindex
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
OnMediaLocation location
Index localtion.
FetcherIndex(const OnMediaLocation &loc)
DefaultIntegral< bool, false > read
Whether we read this index.
Class to encapsulate the OnMediaLocation object and the FileChecker together.
std::list< FileChecker > checkers
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
ZYPP_DECLARE_FLAGS(Flags, Flag)
std::set ordering (less semantic)
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Listentry returned by readdir.