xrootd
XrdClZipArchive.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Michal Simon <michal.simon@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #ifndef SRC_XRDZIP_XRDZIPARCHIVE_HH_
26 #define SRC_XRDZIP_XRDZIPARCHIVE_HH_
27 
28 #include "XrdCl/XrdClFile.hh"
30 #include "XrdCl/XrdClJobManager.hh"
31 #include "XrdCl/XrdClDefaultEnv.hh"
32 #include "XrdCl/XrdClPostMaster.hh"
33 #include "XrdZip/XrdZipEOCD.hh"
34 #include "XrdZip/XrdZipCDFH.hh"
36 #include "XrdZip/XrdZipLFH.hh"
37 #include "XrdCl/XrdClZipCache.hh"
38 
39 #include <memory>
40 #include <unordered_map>
41 
42 //-----------------------------------------------------------------------------
43 // Forward declaration needed for friendship
44 //-----------------------------------------------------------------------------
45 namespace XrdEc{ class StrmWriter; class Reader; template<bool> class OpenOnlyImpl; };
46 class MicroTest;
47 
48 namespace XrdCl
49 {
50 
51  using namespace XrdZip;
52 
53  //---------------------------------------------------------------------------
54  // ZipArchive provides following functionalities:
55  // - parsing of existing ZIP archive
56  // - reading data from existing ZIP archive
57  // - appending data to existing ZIP archive
58  // - querying stat info and checksum for given file in ZIP archive
59  //---------------------------------------------------------------------------
60  class ZipArchive
61  {
62  friend class XrdEc::StrmWriter;
63  friend class XrdEc::Reader;
64  template<bool>
65  friend class XrdEc::OpenOnlyImpl;
66  friend class ::MicroTest;
67 
68  public:
69  //-----------------------------------------------------------------------
71  //-----------------------------------------------------------------------
72  ZipArchive( bool enablePlugIns = true );
73 
74  //-----------------------------------------------------------------------
76  //-----------------------------------------------------------------------
77  virtual ~ZipArchive();
78 
79  //-----------------------------------------------------------------------
87  //-----------------------------------------------------------------------
88  XRootDStatus OpenArchive( const std::string &url,
89  OpenFlags::Flags flags,
90  ResponseHandler *handler,
91  uint16_t timeout = 0 );
92 
93  //-----------------------------------------------------------------------
101  //-----------------------------------------------------------------------
102  XRootDStatus OpenFile( const std::string &fn,
104  uint64_t size = 0,
105  uint32_t crc32 = 0 );
106 
107  //-----------------------------------------------------------------------
116  //-----------------------------------------------------------------------
117  inline
118  XRootDStatus Read( uint64_t offset,
119  uint32_t size,
120  void *buffer,
121  ResponseHandler *handler,
122  uint16_t timeout = 0 )
123  {
124  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
125  return ReadFrom( openfn, offset, size, buffer, handler, timeout );
126  }
127 
128  //-----------------------------------------------------------------------
138  //-----------------------------------------------------------------------
139  XRootDStatus ReadFrom( const std::string &fn,
140  uint64_t offset,
141  uint32_t size,
142  void *buffer,
143  ResponseHandler *handler,
144  uint16_t timeout = 0 );
145 
146  //-----------------------------------------------------------------------
154  //-----------------------------------------------------------------------
155  inline XRootDStatus Write( uint32_t size,
156  const void *buffer,
157  ResponseHandler *handler,
158  uint16_t timeout = 0 )
159  {
160  if( openstage != Done || openfn.empty() )
162  errInvalidOp, "Archive not opened." );
163 
164  return WriteImpl( size, buffer, handler, timeout );
165  }
166 
167  //-----------------------------------------------------------------------
177  //-----------------------------------------------------------------------
178  XRootDStatus AppendFile( const std::string &fn,
179  uint32_t crc32,
180  uint32_t size,
181  const void *buffer,
182  ResponseHandler *handler,
183  uint16_t timeout = 0 );
184 
185  //-----------------------------------------------------------------------
191  //-----------------------------------------------------------------------
192  inline XRootDStatus Stat( const std::string &fn, StatInfo *&info )
193  { // make sure archive has been opened and CD has been parsed
194  if( openstage != Done )
195  return XRootDStatus( stError, errInvalidOp );
196  // make sure the file is part of the archive
197  auto cditr = cdmap.find( fn );
198  if( cditr == cdmap.end() )
199  return XRootDStatus( stError, errNotFound );
200  // create the result
201  info = make_stat( fn );
202  return XRootDStatus();
203  }
204 
205  //-----------------------------------------------------------------------
210  //-----------------------------------------------------------------------
211  inline XRootDStatus Stat( StatInfo *&info )
212  {
213  if( openfn.empty() )
214  return XRootDStatus( stError, errInvalidOp );
215  return Stat( openfn, info );
216  }
217 
218  //-----------------------------------------------------------------------
224  //-----------------------------------------------------------------------
225  inline XRootDStatus GetCRC32( const std::string &fn, uint32_t &cksum )
226  { // make sure archive has been opened and CD has been parsed
227  if( openstage != Done )
228  return XRootDStatus( stError, errInvalidOp );
229  // make sure the file is part of the archive
230  auto cditr = cdmap.find( fn );
231  if( cditr == cdmap.end() )
232  return XRootDStatus( stError, errNotFound );
233  cksum = cdvec[cditr->second]->ZCRC32;
234  return XRootDStatus();
235  }
236 
237  //-----------------------------------------------------------------------
239  //
243  //-----------------------------------------------------------------------
245  uint16_t timeout = 0 );
246 
247  //-----------------------------------------------------------------------
250  //-----------------------------------------------------------------------
252  {
253  if( openstage != Done || openfn.empty() )
255  errInvalidOp, "Archive not opened." );
256  openfn.clear();
257  lfh.reset();
258  return XRootDStatus();
259  }
260 
261  //-----------------------------------------------------------------------
264  //-----------------------------------------------------------------------
265  XRootDStatus List( DirectoryList *&list );
266 
267  //-----------------------------------------------------------------------
269  //-----------------------------------------------------------------------
270  inline bool IsOpen()
271  {
272  return openstage == Done;
273  }
274 
275  //-----------------------------------------------------------------------
277  //-----------------------------------------------------------------------
278  inline bool SetProperty( const std::string &name, const std::string &value )
279  {
280  return archive.SetProperty( name, value );
281  }
282 
283  private:
284 
285  //-----------------------------------------------------------------------
294  //-----------------------------------------------------------------------
295  XRootDStatus WriteImpl( uint32_t size,
296  const void *buffer,
297  ResponseHandler *handler,
298  uint16_t timeout );
299 
300  //-----------------------------------------------------------------------
308  //-----------------------------------------------------------------------
309  XRootDStatus OpenOnly( const std::string &url,
310  bool update,
311  ResponseHandler *handler,
312  uint16_t timeout = 0 );
313 
314  //-----------------------------------------------------------------------
318  //-----------------------------------------------------------------------
319  buffer_t GetCD();
320 
321  //-----------------------------------------------------------------------
325  //-----------------------------------------------------------------------
326  void SetCD( const buffer_t &buffer );
327 
328  //-----------------------------------------------------------------------
333  //-----------------------------------------------------------------------
334  template<typename Response>
335  inline static AnyObject* PkgRsp( Response *rsp )
336  {
337  if( !rsp ) return nullptr;
338  AnyObject *pkg = new AnyObject();
339  pkg->Set( rsp );
340  return pkg;
341  }
342 
343  //-----------------------------------------------------------------------
345  //-----------------------------------------------------------------------
346  template<typename Response>
347  inline static void Free( XRootDStatus *st, Response *rsp )
348  {
349  delete st;
350  delete rsp;
351  }
352 
353  //-----------------------------------------------------------------------
360  //-----------------------------------------------------------------------
361  template<typename Response>
362  inline static void Schedule( ResponseHandler *handler, XRootDStatus *st, Response *rsp = nullptr )
363  {
364  if( !handler ) return Free( st, rsp );
366  if( jobMgr->IsWorker() )
367  // this is a worker thread so we can simply call the handler
368  handler->HandleResponse( st, PkgRsp( rsp ) );
369  else
370  {
371  ResponseJob *job = new ResponseJob( handler, st, PkgRsp( rsp ), 0 );
373  }
374  }
375 
376  //-----------------------------------------------------------------------
382  //-----------------------------------------------------------------------
383  inline static StatInfo* make_stat( const StatInfo &starch, uint64_t size )
384  {
385  StatInfo *info = new StatInfo( starch );
386  uint32_t flags = info->GetFlags();
387  info->SetFlags( flags & ( ~StatInfo::IsWritable ) ); // make sure it is not listed as writable
388  info->SetSize( size );
389  return info;
390  }
391 
392  //-----------------------------------------------------------------------
397  //-----------------------------------------------------------------------
398  inline StatInfo* make_stat( const std::string &fn )
399  {
400  StatInfo *infoptr = 0;
401  XRootDStatus st = archive.Stat( false, infoptr );
402  std::unique_ptr<StatInfo> stinfo( infoptr );
403  auto itr = cdmap.find( fn );
404  if( itr == cdmap.end() ) return nullptr;
405  size_t index = itr->second;
406  return make_stat( *stinfo, cdvec[index]->uncompressedSize );
407  }
408 
409  //-----------------------------------------------------------------------
411  //-----------------------------------------------------------------------
412  inline static XRootDStatus* make_status( const XRootDStatus &status = XRootDStatus() )
413  {
414  return new XRootDStatus( status );
415  }
416 
417  //-----------------------------------------------------------------------
419  //-----------------------------------------------------------------------
420  inline void Clear()
421  {
422  buffer.reset();
423  eocd.reset();
424  cdvec.clear();
425  cdmap.clear();
426  zip64eocd.reset();
427  openstage = None;
428  }
429 
430  //-----------------------------------------------------------------------
432  //-----------------------------------------------------------------------
434  {
435  None = 0, //< opening/parsing not started
436  HaveEocdBlk, //< we have the End of Central Directory record
437  HaveZip64EocdlBlk, //< we have the ZIP64 End of Central Directory locator record
438  HaveZip64EocdBlk, //< we have the ZIP64 End of Central Directory record
439  HaveCdRecords, //< we have Central Directory records
440  Done, //< we are done parsing the Central Directory
441  Error, //< opening/parsing failed
442  NotParsed //< the ZIP archive has been opened but Central Directory is not parsed
443  };
444 
445  //-----------------------------------------------------------------------
447  //-----------------------------------------------------------------------
448  typedef std::unordered_map<std::string, ZipCache> zipcache_t;
449 
450  File archive; //> File object for handling the ZIP archive
451  uint64_t archsize; //> size of the ZIP archive
452  bool cdexists; //> true if Central Directory exists, false otherwise
453  bool updated; //> true if the ZIP archive has been updated, false otherwise
454  std::unique_ptr<char[]> buffer; //> buffer for keeping the data to be parsed or raw data
455  std::unique_ptr<EOCD> eocd; //> End of Central Directory record
456  cdvec_t cdvec; //> vector of Central Directory File Headers
457  cdmap_t cdmap; //> mapping of file name to CDFH index
458  uint64_t cdoff; //> Central Directory offset
459  uint32_t orgcdsz; //> original CD size
460  uint32_t orgcdcnt; //> original number CDFH records
461  buffer_t orgcdbuf; //> buffer with the original CDFH records
462  std::unique_ptr<ZIP64_EOCD> zip64eocd; //> ZIP64 End of Central Directory record
463  OpenStages openstage; //> stage of opening / parsing a ZIP archive
464  std::string openfn; //> file name of opened file
465  zipcache_t zipcache; //> cache for inflating compressed data
466  std::unique_ptr<LFH> lfh; //> Local File Header record for the newly appended file
467  };
468 
469 } /* namespace XrdZip */
470 
471 #endif /* SRC_XRDZIP_XRDZIPARCHIVE_HH_ */
Write access is allowed.
Definition: XrdClXRootDResponses.hh:414
A synchronized queue.
Definition: XrdClJobManager.hh:50
Definition: XrdClAnyObject.hh:32
static void Free(XRootDStatus *st, Response *rsp)
Free status and response.
Definition: XrdClZipArchive.hh:347
buffer_t orgcdbuf
Definition: XrdClZipArchive.hh:461
std::vector< char > buffer_t
Definition: XrdZipUtils.hh:54
Definition: XrdClZipArchive.hh:436
cdvec_t cdvec
Definition: XrdClZipArchive.hh:456
ZipReadFromImpl< false > ReadFrom(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void *> buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:302
Directory list.
Definition: XrdClXRootDResponses.hh:649
Object stat info.
Definition: XrdClXRootDResponses.hh:399
Call the user callback.
Definition: XrdClResponseJob.hh:30
void Clear()
Clear internal ZipArchive objects.
Definition: XrdClZipArchive.hh:420
Definition: XrdClZipArchive.hh:438
StatInfo * make_stat(const std::string &fn)
Definition: XrdClZipArchive.hh:398
Definition: XrdClZipArchive.hh:437
File archive
Definition: XrdClZipArchive.hh:450
XRootDStatus Stat(const std::string &fn, StatInfo *&info)
Definition: XrdClZipArchive.hh:192
std::unordered_map< std::string, ZipCache > zipcache_t
Type that maps file name to its cache.
Definition: XrdClZipArchive.hh:448
Definition: XrdEcStrmWriter.hh:52
XRootDStatus CloseFile()
Definition: XrdClZipArchive.hh:251
static XRootDStatus * make_status(const XRootDStatus &status=XRootDStatus())
Allocate new XRootDStatus object.
Definition: XrdClZipArchive.hh:412
AppendFileImpl< false > AppendFile(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint32_t > crc32, Arg< uint32_t > size, Arg< const void *> buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:421
void QueueJob(Job *job, void *arg=0)
Add a job to be run.
Definition: XrdClJobManager.hh:92
uint32_t GetFlags() const
Get flags.
cdmap_t cdmap
Definition: XrdClZipArchive.hh:457
A file.
Definition: XrdClFile.hh:45
XRootDStatus Write(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:155
std::unique_ptr< ZIP64_EOCD > zip64eocd
Definition: XrdClZipArchive.hh:462
Definition: XrdZipCDFH.hh:39
CloseArchiveImpl< false > CloseArchive(Ctx< ZipArchive > zip, uint16_t timeout=0)
Factory for creating CloseFileImpl objects.
Definition: XrdClZipOperations.hh:621
Definition: XrdClZipArchive.hh:441
zipcache_t zipcache
Definition: XrdClZipArchive.hh:465
none object for initializing empty Optional
Definition: XrdClOptional.hh:35
void Set(Type object, bool own=true)
Definition: XrdClAnyObject.hh:59
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errNotFound
Definition: XrdClStatus.hh:100
Request status.
Definition: XrdClXRootDResponses.hh:218
Definition: XrdClAnyObject.hh:25
StatImpl< false > Stat(Ctx< File > file, Arg< bool > force, uint16_t timeout=0)
Definition: XrdClFileOperations.hh:358
static StatInfo * make_stat(const StatInfo &starch, uint64_t size)
Definition: XrdClZipArchive.hh:383
OpenArchiveImpl< false > OpenArchive(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< OpenFlags::Flags > flags, uint16_t timeout=0)
Factory for creating OpenArchiveImpl objects.
Definition: XrdClZipOperations.hh:123
static void Schedule(ResponseHandler *handler, XRootDStatus *st, Response *rsp=nullptr)
Definition: XrdClZipArchive.hh:362
std::unique_ptr< EOCD > eocd
Definition: XrdClZipArchive.hh:455
std::unordered_map< std::string, size_t > cdmap_t
Definition: XrdZipCDFH.hh:54
uint32_t orgcdsz
Definition: XrdClZipArchive.hh:459
bool SetProperty(const std::string &name, const std::string &value)
Set property on the underlying File object.
Definition: XrdClZipArchive.hh:278
bool IsWorker()
Definition: XrdClJobManager.hh:102
OpenStages
Stages of opening and parsing a ZIP archive.
Definition: XrdClZipArchive.hh:433
Handle an async response.
Definition: XrdClXRootDResponses.hh:1040
virtual void HandleResponse(XRootDStatus *status, AnyObject *response)
Definition: XrdClXRootDResponses.hh:1070
bool IsOpen()
Definition: XrdClZipArchive.hh:270
bool cdexists
Definition: XrdClZipArchive.hh:452
Definition: XrdEcReader.hh:57
uint32_t orgcdcnt
Definition: XrdClZipArchive.hh:460
void SetFlags(uint32_t flags)
Set flags.
Definition: XrdClZipArchive.hh:439
JobManager * GetJobManager()
Get the job manager object user by the post master.
XRootDStatus GetCRC32(const std::string &fn, uint32_t &cksum)
Definition: XrdClZipArchive.hh:225
XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:118
Definition: XrdClZipArchive.hh:45
Definition: XrdClZipArchive.hh:60
OpenFileImpl< false > OpenFile(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< OpenFlags::Flags > flags=OpenFlags::None, Arg< uint64_t > size=0, Arg< uint32_t > crc32=0, uint16_t timeout=0)
Factory for creating OpenFileImpl objects.
Definition: XrdClZipOperations.hh:184
Nothing.
Definition: XrdClFileSystem.hh:77
std::vector< std::unique_ptr< CDFH > > cdvec_t
Definition: XrdZipCDFH.hh:44
std::string openfn
Definition: XrdClZipArchive.hh:464
OpenStages openstage
Definition: XrdClZipArchive.hh:463
static PostMaster * GetPostMaster()
Get default post master.
Flags
Open flags, may be or&#39;d when appropriate.
Definition: XrdClFileSystem.hh:75
void SetSize(uint64_t size)
Set size.
XRootDStatus Stat(StatInfo *&info)
Definition: XrdClZipArchive.hh:211
Write operation (.
Definition: XrdClFileOperations.hh:367
Definition: XrdClZipArchive.hh:440
uint64_t cdoff
Definition: XrdClZipArchive.hh:458
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:466
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:51
std::unique_ptr< char[]> buffer
Definition: XrdClZipArchive.hh:454
uint64_t archsize
Definition: XrdClZipArchive.hh:451
Definition: XrdClZipArchive.hh:45
bool updated
Definition: XrdClZipArchive.hh:453
ZipListImpl< false > List(Ctx< ZipArchive > zip)
Factory for creating ZipStatImpl objects.
Definition: XrdClZipOperations.hh:574
static AnyObject * PkgRsp(Response *rsp)
Definition: XrdClZipArchive.hh:335