xrootd
XrdPfcInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDPFC_INFO_HH__
2 #define __XRDPFC_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <stdio.h>
22 #include <time.h>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 #include "XrdPfcTypes.hh"
31 
32 class XrdOssDF;
33 class XrdCksCalc;
34 class XrdSysTrace;
35 
36 namespace XrdPfc
37 {
38 class Stats;
39 
40 //----------------------------------------------------------------------------
42 //----------------------------------------------------------------------------
43 
44 class Info
45 {
46 public:
47  struct Status {
48  union {
49  struct {
50  int f_cksum_check : 3;
51 
52  int _free_bits_ : 29;
53  };
54  unsigned int _raw_;
55  };
56  Status() : _raw_(0) {}
57  };
58 
60  struct AStat
61  {
62  time_t AttachTime;
63  time_t DetachTime;
64  int NumIos;
65  int Duration;
66  int NumMerged;
67  int Reserved;
68  long long BytesHit;
69  long long BytesMissed;
70  long long BytesBypassed;
71 
72  AStat() :
73  AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), Reserved(0),
75  {}
76 
77  void MergeWith(const AStat &a);
78  };
79 
80  struct Store
81  {
82  long long m_buffer_size;
83  long long m_file_size;
84  time_t m_creationTime;
85  time_t m_noCkSumTime;
86  size_t m_accessCnt;
89 
90  Store () :
93  {}
94  };
95 
96 
97  //------------------------------------------------------------------------
99  //------------------------------------------------------------------------
100  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
101 
102  //------------------------------------------------------------------------
104  //------------------------------------------------------------------------
105  ~Info();
106 
107  //---------------------------------------------------------------------
109  //---------------------------------------------------------------------
110  void SetBitWritten(int i);
111 
112  //---------------------------------------------------------------------
114  //---------------------------------------------------------------------
115  bool TestBitWritten(int i) const;
116 
117  //---------------------------------------------------------------------
119  //---------------------------------------------------------------------
120  bool TestBitPrefetch(int i) const;
121 
122  //---------------------------------------------------------------------
124  //---------------------------------------------------------------------
125  void SetBitPrefetch(int i);
126 
127  //---------------------------------------------------------------------
129  //---------------------------------------------------------------------
130  void SetBitSynced(int i);
131 
132  //---------------------------------------------------------------------
134  //---------------------------------------------------------------------
135  void SetAllBitsSynced();
136 
137  void SetBufferSize(long long);
138 
139  void SetFileSizeAndCreationTime(long long);
140 
141  //---------------------------------------------------------------------
143  //---------------------------------------------------------------------
144  void ResizeBits();
145 
146  //---------------------------------------------------------------------
152  //---------------------------------------------------------------------
153  bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0);
154 
155  //---------------------------------------------------------------------
161  //---------------------------------------------------------------------
162  bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0);
163 
164  //---------------------------------------------------------------------
166  //---------------------------------------------------------------------
168 
169  //---------------------------------------------------------------------
171  //---------------------------------------------------------------------
172  void ResetAllAccessStats();
173 
174  //---------------------------------------------------------------------
176  //---------------------------------------------------------------------
177  void WriteIOStatAttach();
178 
179  //---------------------------------------------------------------------
181  //---------------------------------------------------------------------
182  void WriteIOStat(Stats& s);
183 
184  //---------------------------------------------------------------------
186  //---------------------------------------------------------------------
187  void WriteIOStatDetach(Stats& s);
188 
189  //---------------------------------------------------------------------
191  //---------------------------------------------------------------------
192  void WriteIOStatSingle(long long bytes_disk);
193 
194  //---------------------------------------------------------------------
196  //---------------------------------------------------------------------
197  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
198 
199  //---------------------------------------------------------------------
201  //---------------------------------------------------------------------
202  bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const;
203 
204  //---------------------------------------------------------------------
206  //---------------------------------------------------------------------
207  int GetBitvecSizeInBytes() const;
208 
209  //---------------------------------------------------------------------
211  //---------------------------------------------------------------------
212  int GetNBlocks() const;
213 
214  //---------------------------------------------------------------------
216  //---------------------------------------------------------------------
217  long long GetFileSize() const;
218 
219  //---------------------------------------------------------------------
221  //---------------------------------------------------------------------
222  bool GetLatestDetachTime(time_t& t) const;
223 
224  //---------------------------------------------------------------------
226  //---------------------------------------------------------------------
227  const AStat* GetLastAccessStats() const;
228 
229  //---------------------------------------------------------------------
231  //---------------------------------------------------------------------
232  long long GetBufferSize() const;
233 
234  //---------------------------------------------------------------------
236  //---------------------------------------------------------------------
237  bool IsComplete() const;
238 
239  //---------------------------------------------------------------------
241  //---------------------------------------------------------------------
242  int GetNDownloadedBlocks() const;
243 
244  //---------------------------------------------------------------------
246  //---------------------------------------------------------------------
247  long long GetNDownloadedBytes() const;
248 
249  //---------------------------------------------------------------------
251  //---------------------------------------------------------------------
252  int GetLastDownloadedBlock() const;
253 
254  //---------------------------------------------------------------------
256  //---------------------------------------------------------------------
257  long long GetExpectedDataFileSize() const;
258 
259  //---------------------------------------------------------------------
261  //---------------------------------------------------------------------
263 
264  //---------------------------------------------------------------------
266  //---------------------------------------------------------------------
267  size_t GetAccessCnt() const { return m_store.m_accessCnt; }
268 
269  //---------------------------------------------------------------------
271  //---------------------------------------------------------------------
272  int GetVersion() { return m_version; }
273 
274  //---------------------------------------------------------------------
276  //---------------------------------------------------------------------
277  const Store& RefStoredData() const { return m_store; }
278  const std::vector<AStat>& RefAStats() const { return m_astats; }
279 
280  //---------------------------------------------------------------------
282  //---------------------------------------------------------------------
283  time_t GetCreationTime() const { return m_store.m_creationTime; }
284 
285  //---------------------------------------------------------------------
287  //---------------------------------------------------------------------
288  uint32_t CalcCksumStore();
289  uint32_t CalcCksumSyncedAndAStats();
290  void CalcCksumMd5(unsigned char* buff, char* digest);
291 
293  const char* GetCkSumStateAsText() const;
294 
296  bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; }
299 
302  void ResetCkSumCache();
303  void ResetCkSumNet();
304 
305  bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; }
306  time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; }
308 
309 #ifdef XRDPFC_CKSUM_TEST
310  static void TestCksumStuff();
311 #endif
312 
313  static const char* m_traceID; // has to be m_ (convention in TRACE macros)
314  static const char* s_infoExtension;
315  static const size_t s_infoExtensionLen;
316  static size_t s_maxNumAccess; // can be set from configuration
317  static const int s_defaultVersion;
318 
319  XrdSysTrace* GetTrace() const {return m_trace; }
320 
321 protected:
323 
325  unsigned char *m_buff_synced;
326  unsigned char *m_buff_written;
327  unsigned char *m_buff_prefetch;
328  std::vector<AStat> m_astats;
329 
332  bool m_complete;
334 
335 private:
336  inline unsigned char cfiBIT(int n) const { return 1 << n; }
337 
338  // Reading functions for older cinfo file formats
339  bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
340  bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
341 
343 };
344 
345 //------------------------------------------------------------------------------
346 
347 inline bool Info::TestBitWritten(int i) const
348 {
349  const int cn = i/8;
350  assert(cn < GetBitvecSizeInBytes());
351 
352  const int off = i - cn*8;
353  return (m_buff_written[cn] & cfiBIT(off)) != 0;
354 }
355 
356 inline void Info::SetBitWritten(int i)
357 {
358  const int cn = i/8;
359  assert(cn < GetBitvecSizeInBytes());
360 
361  const int off = i - cn*8;
362  m_buff_written[cn] |= cfiBIT(off);
363 }
364 
365 inline void Info::SetBitPrefetch(int i)
366 {
367  if (!m_buff_prefetch) return;
368 
369  const int cn = i/8;
370  assert(cn < GetBitvecSizeInBytes());
371 
372  const int off = i - cn*8;
373  m_buff_prefetch[cn] |= cfiBIT(off);
374 }
375 
376 inline bool Info::TestBitPrefetch(int i) const
377 {
378  if (!m_buff_prefetch) return false;
379 
380  const int cn = i/8;
381  assert(cn < GetBitvecSizeInBytes());
382 
383  const int off = i - cn*8;
384  return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
385 }
386 
387 inline void Info::SetBitSynced(int i)
388 {
389  const int cn = i/8;
390  assert(cn < GetBitvecSizeInBytes());
391 
392  const int off = i - cn*8;
393  m_buff_synced[cn] |= cfiBIT(off);
394 }
395 
396 //------------------------------------------------------------------------------
397 
398 inline int Info::GetNDownloadedBlocks() const
399 {
400  int cntd = 0;
401  for (int i = 0; i < m_bitvecSizeInBits; ++i)
402  if (TestBitWritten(i)) cntd++;
403 
404  return cntd;
405 }
406 
407 inline long long Info::GetNDownloadedBytes() const
408 {
410 }
411 
413 {
414  for (int i = m_bitvecSizeInBits - 1; i >= 0; --i)
415  if (TestBitWritten(i)) return i;
416 
417  return -1;
418 }
419 
420 inline long long Info::GetExpectedDataFileSize() const
421 {
422  int last_block = GetLastDownloadedBlock();
423  if (last_block == m_bitvecSizeInBits - 1)
424  return m_store.m_file_size;
425  else
426  return (last_block + 1) * m_store.m_buffer_size;
427 }
428 
429 inline int Info::GetBitvecSizeInBytes() const
430 {
431  if (m_bitvecSizeInBits)
432  return ((m_bitvecSizeInBits - 1)/8 + 1);
433  else
434  return 0;
435 }
436 
437 inline int Info::GetNBlocks() const
438 {
439  return m_bitvecSizeInBits;
440 }
441 
442 inline long long Info::GetFileSize() const
443 {
444  return m_store.m_file_size;
445 }
446 
447 inline bool Info::IsComplete() const
448 {
449  return m_complete;
450 }
451 
452 inline bool Info::IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
453 {
454  // TODO rewrite to use full byte comparisons outside of edges ?
455  // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits.
456  for (int i = firstIdx; i < lastIdx; ++i)
457  if (! TestBitWritten(i)) return true;
458 
459  return false;
460 }
461 
463 {
465 }
466 
467 inline long long Info::GetBufferSize() const
468 {
469  return m_store.m_buffer_size;
470 }
471 
472 }
473 #endif
void ResizeBits()
Reserve bit vectors for file_size / buffer_size bytes.
const Store & RefStoredData() const
Get stored data.
Definition: XrdPfcInfo.hh:277
void MergeWith(const AStat &a)
size_t GetAccessCnt() const
Get number of accesses.
Definition: XrdPfcInfo.hh:267
int NumIos
number of IO objects attached during this access
Definition: XrdPfcInfo.hh:64
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:316
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
Definition: XrdPfcInfo.hh:365
unsigned char * m_buff_written
download state vector
Definition: XrdPfcInfo.hh:326
int GetBitvecSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdPfcInfo.hh:429
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:314
int NumMerged
number of times the record has been merged
Definition: XrdPfcInfo.hh:66
XrdSysTrace * GetTrace() const
Definition: XrdPfcInfo.hh:319
int Reserved
reserved / alignment
Definition: XrdPfcInfo.hh:67
bool IsCkSumBoth() const
Definition: XrdPfcInfo.hh:298
int GetNBlocks() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdPfcInfo.hh:437
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdPfcInfo.hh:407
void SetBitWritten(int i)
Mark block as written to disk.
Definition: XrdPfcInfo.hh:356
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdPfcInfo.hh:462
time_t m_noCkSumTime
time when first non-cksummed block was detected
Definition: XrdPfcInfo.hh:85
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:44
Access statistics.
Definition: XrdPfcInfo.hh:60
time_t GetNoCkSumTime() const
Definition: XrdPfcInfo.hh:306
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdPfcInfo.hh:420
long long BytesMissed
read from remote and cached
Definition: XrdPfcInfo.hh:69
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
unsigned char cfiBIT(int n) const
Definition: XrdPfcInfo.hh:336
void SetAllBitsSynced()
Mark all blocks as synced to disk.
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdPfcInfo.hh:327
const AStat * GetLastAccessStats() const
Get latest access stats.
unsigned int _raw_
Definition: XrdPfcInfo.hh:54
int m_astatSize
size of AStat vector
Definition: XrdPfcInfo.hh:88
int Duration
total duration of all IOs attached
Definition: XrdPfcInfo.hh:65
void CalcCksumMd5(unsigned char *buff, char *digest)
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
Definition: XrdPfcInfo.hh:376
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdPfcInfo.hh:412
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdPfcInfo.hh:398
Definition: XrdPfcInfo.hh:47
int _free_bits_
Definition: XrdPfcInfo.hh:52
Definition: XrdSysTrace.hh:48
Definition: XrdPfc.hh:40
long long GetFileSize() const
Get file size.
Definition: XrdPfcInfo.hh:442
CkSumCheck_e GetCkSumState() const
Definition: XrdPfcInfo.hh:292
bool Write(XrdOssDF *fp, const char *dname, const char *fname=0)
int GetVersion()
Get version.
Definition: XrdPfcInfo.hh:272
Store m_store
Definition: XrdPfcInfo.hh:324
bool ReadV3(XrdOssDF *fp, off_t off, const char *dname, const char *fname)
std::vector< AStat > m_astats
access records
Definition: XrdPfcInfo.hh:328
void ResetCkSumNet()
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
time_t AttachTime
open time
Definition: XrdPfcInfo.hh:62
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdPfcInfo.hh:467
Definition: XrdCksCalc.hh:39
const char * GetCkSumStateAsText() const
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
long long BytesBypassed
read from remote and dropped
Definition: XrdPfcInfo.hh:70
long long m_file_size
size of file in bytes
Definition: XrdPfcInfo.hh:83
bool IsCkSumNet() const
Definition: XrdPfcInfo.hh:296
void SetFileSizeAndCreationTime(long long)
long long BytesHit
read from cache
Definition: XrdPfcInfo.hh:68
unsigned char * m_buff_synced
disk written state vector
Definition: XrdPfcInfo.hh:325
long long m_buffer_size
buffer / block size
Definition: XrdPfcInfo.hh:82
bool IsCkSumCache() const
Definition: XrdPfcInfo.hh:295
static const size_t s_infoExtensionLen
Definition: XrdPfcInfo.hh:315
void SetBitSynced(int i)
Mark block as synced to disk.
Definition: XrdPfcInfo.hh:387
CkSumCheck_e
Definition: XrdPfcTypes.hh:23
time_t DetachTime
close time
Definition: XrdPfcInfo.hh:63
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdPfcInfo.hh:333
XrdSysTrace * m_trace
Definition: XrdPfcInfo.hh:322
static const char * m_traceID
Definition: XrdPfcInfo.hh:313
Store()
Definition: XrdPfcInfo.hh:90
void SetCkSumState(CkSumCheck_e css)
Definition: XrdPfcInfo.hh:300
Statistics of cache utilisation by a File object.
Definition: XrdPfcStats.hh:30
bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdPfcInfo.hh:452
Status()
Definition: XrdPfcInfo.hh:56
void ResetAllAccessStats()
Reset IO Stats.
int m_bitvecSizeInBits
cached
Definition: XrdPfcInfo.hh:331
bool IsCkSumAny() const
Definition: XrdPfcInfo.hh:297
size_t m_accessCnt
total access count for the file
Definition: XrdPfcInfo.hh:86
bool m_complete
cached
Definition: XrdPfcInfo.hh:332
void SetBufferSize(long long)
Definition: XrdPfcTypes.hh:23
bool HasNoCkSumTime() const
Definition: XrdPfcInfo.hh:305
Definition: XrdOss.hh:62
time_t GetCreationTime() const
Get file size.
Definition: XrdPfcInfo.hh:283
const std::vector< AStat > & RefAStats() const
Definition: XrdPfcInfo.hh:278
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
Definition: XrdPfcTypes.hh:23
void ResetCkSumCache()
Definition: XrdPfcTypes.hh:23
~Info()
Destructor.
Status m_status
status information
Definition: XrdPfcInfo.hh:87
int f_cksum_check
as in enum CkSumCheck_e
Definition: XrdPfcInfo.hh:50
time_t GetNoCkSumTimeForUVKeep() const
Definition: XrdPfcInfo.hh:307
int m_version
Definition: XrdPfcInfo.hh:330
Definition: XrdPfcInfo.hh:80
time_t m_creationTime
time the info file was created
Definition: XrdPfcInfo.hh:84
uint32_t CalcCksumStore()
Get cksum, MD5 is for backward compatibility with V2 and V3.
AStat()
Definition: XrdPfcInfo.hh:72
uint32_t CalcCksumSyncedAndAStats()
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
Definition: XrdPfcInfo.hh:347
bool ReadV2(XrdOssDF *fp, off_t off, const char *dname, const char *fname)
void DowngradeCkSumState(CkSumCheck_e css_ref)
Definition: XrdPfcInfo.hh:301
XrdCksCalc * m_cksCalcMd5
Definition: XrdPfcInfo.hh:342
static const int s_defaultVersion
Definition: XrdPfcInfo.hh:317
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
bool IsComplete() const
Get complete status.
Definition: XrdPfcInfo.hh:447