Review Board 1.7.22


HBASE-5625 Avoid byte buffer allocations when reading a value from a Result object

Review Request #4607 - Created April 2, 2012 and updated

Tudor Scurtu
trunk
HBASE-5625
Reviewers
hbase
hbase-git
When calling Result.getValue(), an extra dummy KeyValue and its associated underlying byte array are allocated, as well as a persistent buffer that will contain the returned value.

These can be avoided by reusing a static array for the dummy object and by passing a ByteBuffer object as a value destination buffer to the read method.
Added value check to TestResult#testBasic and TestResult.testMultiVersion.

Diff revision 1

This is not the most recent revision of the diff. The latest diff is revision 4. See what's changed.

1 2 3 4
1 2 3 4

  1. src/main/java/org/apache/hadoop/hbase/KeyValue.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/client/Result.java: Loading...
  3. src/test/java/org/apache/hadoop/hbase/client/TestResult.java: Loading...
src/main/java/org/apache/hadoop/hbase/KeyValue.java
Revision 243d76f New Change
[20] 19 lines
[+20]
20
package org.apache.hadoop.hbase;
20
package org.apache.hadoop.hbase;
21

    
   
21

   
22
import java.io.DataInput;
22
import java.io.DataInput;
23
import java.io.DataOutput;
23
import java.io.DataOutput;
24
import java.io.IOException;
24
import java.io.IOException;

    
   
25
import java.nio.BufferOverflowException;
25
import java.nio.ByteBuffer;
26
import java.nio.ByteBuffer;
26
import java.util.Comparator;
27
import java.util.Comparator;
27
import java.util.HashMap;
28
import java.util.HashMap;
28
import java.util.Map;
29
import java.util.Map;
29

    
   
30

   
[+20] [20] 473 lines
[+20] [+] static byte[] createEmptyByteArray(final int rlength, int flength,
503
    pos = Bytes.putByte(bytes, pos, type.getCode());
504
    pos = Bytes.putByte(bytes, pos, type.getCode());
504
    return bytes;
505
    return bytes;
505
  }
506
  }
506

    
   
507

   
507
  /**
508
  /**
508
   * Write KeyValue format into a byte array.
509
   * Constructs KeyValue structure filled with specified values. Uses the provided buffer as the

    
   
510
   * data buffer.

    
   
511
   * <p>

    
   
512
   * Column is split into two fields, family and qualifier.
509
   *
513
   *

    
   
514
   * @param buffer the bytes buffer to use
510
   * @param row row key
515
   * @param row row key
511
   * @param roffset row offset
516
   * @param roffset row offset
512
   * @param rlength row length
517
   * @param rlength row length
513
   * @param family family name
518
   * @param family family name
514
   * @param foffset family offset
519
   * @param foffset family offset
[+20] [20] 4 lines
[+20] static byte[] createEmptyByteArray(final int rlength, int flength,
519
   * @param timestamp version timestamp
524
   * @param timestamp version timestamp
520
   * @param type key type
525
   * @param type key type
521
   * @param value column value
526
   * @param value column value
522
   * @param voffset value offset
527
   * @param voffset value offset
523
   * @param vlength value length
528
   * @param vlength value length
524
   * @return The newly created byte array.
529
   * @throws IllegalArgumentException
525
   */
530
   */
526
  static byte [] createByteArray(final byte [] row, final int roffset,
531
  public KeyValue(byte [] buffer,
527
      final int rlength, final byte [] family, final int foffset, int flength,
532
                  final byte [] row, final int roffset, final int rlength,
528
      final byte [] qualifier, final int qoffset, int qlength,
533
                  final byte [] family, final int foffset, final int flength,

    
   
534
                  final byte [] qualifier, final int qoffset, final int qlength,
529
      final long timestamp, final Type type,
535
                  final long timestamp, final Type type,
530
      final byte [] value, final int voffset, int vlength) {
536
                  final byte [] value, final int voffset, final int vlength) {

    
   
537

   

    
   
538
    this.bytes  = buffer;

    
   
539
    this.length = writeByteArray(buffer, row, roffset, rlength,

    
   
540
                                 family, foffset, flength, qualifier, qoffset, qlength,

    
   
541
                                 timestamp, type, value, voffset, vlength);

    
   
542
    this.offset = 0;

    
   
543
  }

    
   
544

   

    
   
545
  /**

    
   
546
   * Checks the parameters passed to a constructor.

    
   
547
   *

    
   
548
   * @param row row key

    
   
549
   * @param rlength row length

    
   
550
   * @param family family name

    
   
551
   * @param flength family length

    
   
552
   * @param qualifier column qualifier

    
   
553
   * @param qlength qualifier length

    
   
554
   * @param value column value

    
   
555
   * @param vlength value length

    
   
556
   *

    
   
557
   * @throws IllegalArgumentException an illegal value was passed

    
   
558
   */

    
   
559
  private static void checkParameters(final byte [] row, final int rlength,

    
   
560
                                      final byte [] family, int flength,

    
   
561
                                      final byte [] qualifier, int qlength,

    
   
562
                                      final byte [] value, int vlength)

    
   
563
          throws IllegalArgumentException {

    
   
564

   
531
    if (rlength > Short.MAX_VALUE) {
565
    if (rlength > Short.MAX_VALUE) {
532
      throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
566
      throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
533
    }
567
    }
534
    if (row == null) {
568
    if (row == null) {
535
      throw new IllegalArgumentException("Row is null");
569
      throw new IllegalArgumentException("Row is null");
[+20] [20] 7 lines
[+20] static byte[] createEmptyByteArray(final int rlength, int flength, [+] private static void checkParameters(final byte [] row, final int rlength,
543
    qlength = qualifier == null ? 0 : qlength;
577
    qlength = qualifier == null ? 0 : qlength;
544
    if (qlength > Integer.MAX_VALUE - rlength - flength) {
578
    if (qlength > Integer.MAX_VALUE - rlength - flength) {
545
      throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
579
      throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
546
    }
580
    }
547
    // Key length
581
    // Key length
548
    long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
582
    long longKeyLength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
549
    if (longkeylength > Integer.MAX_VALUE) {
583
    if (longKeyLength > Integer.MAX_VALUE) {
550
      throw new IllegalArgumentException("keylength " + longkeylength + " > " +
584
      throw new IllegalArgumentException("keylength " + longKeyLength + " > " + Integer.MAX_VALUE);
551
        Integer.MAX_VALUE);

   
552
    }
585
    }
553
    int keylength = (int)longkeylength;

   
554
    // Value length
586
    // Value length
555
    vlength = value == null? 0 : vlength;
587
    vlength = value == null? 0 : vlength;
556
    if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
588
    if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
557
      throw new IllegalArgumentException("Valuer > " +
589
      throw new IllegalArgumentException("Value > " + HConstants.MAXIMUM_VALUE_LENGTH);
558
          HConstants.MAXIMUM_VALUE_LENGTH);
590
    }
559
    }
591
  }
560

    
   
592

   

    
   
593
  /**

    
   
594
   * Write KeyValue format into the provided byte array.

    
   
595
   *

    
   
596
   * @param buffer the bytes buffer to use

    
   
597
   * @param row row key

    
   
598
   * @param roffset row offset

    
   
599
   * @param rlength row length

    
   
600
   * @param family family name

    
   
601
   * @param foffset family offset

    
   
602
   * @param flength family length

    
   
603
   * @param qualifier column qualifier

    
   
604
   * @param qoffset qualifier offset

    
   
605
   * @param qlength qualifier length

    
   
606
   * @param timestamp version timestamp

    
   
607
   * @param type key type

    
   
608
   * @param value column value

    
   
609
   * @param voffset value offset

    
   
610
   * @param vlength value length

    
   
611
   *

    
   
612
   * @return The number of useful bytes in the buffer.

    
   
613
   */

    
   
614
  static int writeByteArray(byte [] buffer,

    
   
615
                            final byte [] row, final int roffset, final int rlength,

    
   
616
                            final byte [] family, final int foffset, int flength,

    
   
617
                            final byte [] qualifier, final int qoffset, int qlength,

    
   
618
                            final long timestamp, final Type type,

    
   
619
                            final byte [] value, final int voffset, int vlength) {

    
   
620

   

    
   
621
    checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);

    
   
622

   

    
   
623
    int keyLength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;

    
   
624
    int keyValueLength = KEYVALUE_INFRASTRUCTURE_SIZE + keyLength + vlength;

    
   
625
    if (keyValueLength > buffer.length) {

    
   
626
      throw new IllegalArgumentException("Buffer size " + buffer.length + " < " + keyValueLength);

    
   
627
    }

    
   
628

   

    
   
629
    // Write key, value and key row length.

    
   
630
    int pos = 0;

    
   
631
    pos = Bytes.putInt(buffer, pos, keyLength);

    
   
632
    pos = Bytes.putInt(buffer, pos, vlength);

    
   
633
    pos = Bytes.putShort(buffer, pos, (short)(rlength & 0x0000ffff));

    
   
634
    pos = Bytes.putBytes(buffer, pos, row, roffset, rlength);

    
   
635
    pos = Bytes.putByte(buffer, pos, (byte) (flength & 0x0000ff));

    
   
636
    if (flength != 0) {

    
   
637
      pos = Bytes.putBytes(buffer, pos, family, foffset, flength);

    
   
638
    }

    
   
639
    if (qlength != 0) {

    
   
640
      pos = Bytes.putBytes(buffer, pos, qualifier, qoffset, qlength);

    
   
641
    }

    
   
642
    pos = Bytes.putLong(buffer, pos, timestamp);

    
   
643
    pos = Bytes.putByte(buffer, pos, type.getCode());

    
   
644
    if (value != null && value.length > 0) {

    
   
645
      pos = Bytes.putBytes(buffer, pos, value, voffset, vlength);

    
   
646
    }

    
   
647

   

    
   
648
    return keyValueLength;

    
   
649
  }

    
   
650

   

    
   
651
  /**

    
   
652
   * Write KeyValue format into a byte array.

    
   
653
   *

    
   
654
   * @param row row key

    
   
655
   * @param roffset row offset

    
   
656
   * @param rlength row length

    
   
657
   * @param family family name

    
   
658
   * @param foffset family offset

    
   
659
   * @param flength family length

    
   
660
   * @param qualifier column qualifier

    
   
661
   * @param qoffset qualifier offset

    
   
662
   * @param qlength qualifier length

    
   
663
   * @param timestamp version timestamp

    
   
664
   * @param type key type

    
   
665
   * @param value column value

    
   
666
   * @param voffset value offset

    
   
667
   * @param vlength value length

    
   
668
   * @return The newly created byte array.

    
   
669
   */

    
   
670
  static byte [] createByteArray(final byte [] row, final int roffset,

    
   
671
      final int rlength, final byte [] family, final int foffset, int flength,

    
   
672
      final byte [] qualifier, final int qoffset, int qlength,

    
   
673
      final long timestamp, final Type type,

    
   
674
      final byte [] value, final int voffset, int vlength) {

    
   
675

   

    
   
676
    checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);

    
   
677

   
561
    // Allocate right-sized byte array.
678
    // Allocate right-sized byte array.
562
    byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength];
679
    int keyLength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;

    
   
680
    byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keyLength + vlength];
563
    // Write key, value and key row length.
681
    // Write key, value and key row length.
564
    int pos = 0;
682
    int pos = 0;
565
    pos = Bytes.putInt(bytes, pos, keylength);
683
    pos = Bytes.putInt(bytes, pos, keyLength);
566
    pos = Bytes.putInt(bytes, pos, vlength);
684
    pos = Bytes.putInt(bytes, pos, vlength);
567
    pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
685
    pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
568
    pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
686
    pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
569
    pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
687
    pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
570
    if(flength != 0) {
688
    if(flength != 0) {
[+20] [20] 434 lines
[+20] [+] public boolean updateLatestStamp(final byte [] now) {
1005
    System.arraycopy(getBuffer(), o, result, 0, l);
1123
    System.arraycopy(getBuffer(), o, result, 0, l);
1006
    return result;
1124
    return result;
1007
  }
1125
  }
1008

    
   
1126

   
1009
  /**
1127
  /**

    
   
1128
   * Loads this object's value into the provided <code>ByteBuffer</code>.

    
   
1129
   * <p>

    
   
1130
   * Does not clear or flip the buffer.

    
   
1131
   *

    
   
1132
   * @param dst the buffer where to write the value

    
   
1133
   *

    
   
1134
   * @throws BufferOverflowException there is insufficient space remaining in the buffer

    
   
1135
   */

    
   
1136
  public void loadValue(ByteBuffer dst)

    
   
1137
          throws BufferOverflowException {

    
   
1138
    dst.put(getBuffer(), getValueOffset(), getValueLength());

    
   
1139
  }

    
   
1140

   

    
   
1141
  /**
1010
   * Primarily for use client-side.  Returns the row of this KeyValue in a new
1142
   * Primarily for use client-side.  Returns the row of this KeyValue in a new
1011
   * byte array.<p>
1143
   * byte array.<p>
1012
   *
1144
   *
1013
   * If server-side, use {@link #getBuffer()} with appropriate offsets and
1145
   * If server-side, use {@link #getBuffer()} with appropriate offsets and
1014
   * lengths instead.
1146
   * lengths instead.
[+20] [20] 278 lines
[+20] [+] public boolean matchingColumn(final byte[] family, final byte[] qualifier) {
1293
    return Bytes.equals(qualifier, 0, qualifier.length,
1425
    return Bytes.equals(qualifier, 0, qualifier.length,
1294
        this.bytes, o + fl, ql);
1426
        this.bytes, o + fl, ql);
1295
  }
1427
  }
1296

    
   
1428

   
1297
  /**
1429
  /**

    
   
1430
   * Checks if column matches.

    
   
1431
   *

    
   
1432
   * @param family family name

    
   
1433
   * @param foffset family offset

    
   
1434
   * @param flength family length

    
   
1435
   * @param qualifier column qualifier

    
   
1436
   * @param qoffset qualifier offset

    
   
1437
   * @param qlength qualifier length

    
   
1438
   *

    
   
1439
   * @return True if column matches

    
   
1440
   */

    
   
1441
  public boolean matchingColumn(final byte [] family,    final int foffset, final int flength,

    
   
1442
                                final byte [] qualifier, final int qoffset, final int qlength) {

    
   
1443
    int rl = getRowLength();

    
   
1444
    int o  = getFamilyOffset(rl);

    
   
1445
    int fl = getFamilyLength(o);

    
   
1446
    int ql = getQualifierLength(rl,fl);

    
   
1447

   

    
   
1448
    if (!Bytes.equals(family, foffset, flength, this.bytes, o, fl)) {

    
   
1449
      return false;

    
   
1450
    }

    
   
1451
    if (qualifier == null || qlength == 0) {

    
   
1452
      return (ql == 0);

    
   
1453
    }

    
   
1454
    return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql);

    
   
1455
  }

    
   
1456

   

    
   
1457
  /**
1298
   * @param left
1458
   * @param left
1299
   * @param loffset
1459
   * @param loffset
1300
   * @param llength
1460
   * @param llength
1301
   * @param lfamilylength Offset of family delimiter in left column.
1461
   * @param lfamilylength Offset of family delimiter in left column.
1302
   * @param right
1462
   * @param right
[+20] [20] 514 lines
[+20] [+] public static KeyValue createFirstOnRow(final byte [] row,
1817
    return new KeyValue(row, roffset, rlength, family,
1977
    return new KeyValue(row, roffset, rlength, family,
1818
        foffset, flength, qualifier, qoffset, qlength,
1978
        foffset, flength, qualifier, qoffset, qlength,
1819
        HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
1979
        HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
1820
  }
1980
  }
1821

    
   
1981

   

    
   
1982

   

    
   
1983
  /**

    
   
1984
   * Create a KeyValue for the specified row, family and qualifier that would be

    
   
1985
   * smaller than all other possible KeyValues that have the same row,

    
   
1986
   * family, qualifier.

    
   
1987
   * Used for seeking.

    
   
1988
   *

    
   
1989
   * @param buffer the buffer to use for the new <code>KeyValue</code> object

    
   
1990
   * @param row the value key

    
   
1991
   * @param family family name

    
   
1992
   * @param qualifier column qualifier

    
   
1993
   *

    
   
1994
   * @return First possible key on passed Row, Family, Qualifier.

    
   
1995
   *

    
   
1996
   * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger

    
   
1997
   * than the provided buffer or than <code>Integer.MAX_VALUE</code>

    
   
1998
   */

    
   
1999
  public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,

    
   
2000
                                          final byte [] family, final byte [] qualifier)

    
   
2001
          throws IllegalArgumentException {

    
   
2002

   

    
   
2003
    return createFirstOnRow(buffer, row, family, 0, family.length, qualifier, 0, qualifier.length);

    
   
2004
  }

    
   
2005

   

    
   
2006
  /**

    
   
2007
   * Create a KeyValue for the specified row, family and qualifier that would be

    
   
2008
   * smaller than all other possible KeyValues that have the same row,

    
   
2009
   * family, qualifier.

    
   
2010
   * Used for seeking.

    
   
2011
   *

    
   
2012
   * @param buffer the buffer to use for the new <code>KeyValue</code> object

    
   
2013
   * @param row the value key

    
   
2014
   * @param family family name

    
   
2015
   * @param foffset family offset

    
   
2016
   * @param flength family length

    
   
2017
   * @param qualifier column qualifier

    
   
2018
   * @param qoffset qualifier offset

    
   
2019
   * @param qlength qualifier length

    
   
2020
   *

    
   
2021
   * @return First possible key on passed Row, Family, Qualifier.

    
   
2022
   *

    
   
2023
   * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger

    
   
2024
   * than the provided buffer or than <code>Integer.MAX_VALUE</code>

    
   
2025
   */

    
   
2026
  public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,

    
   
2027
      final byte [] family,    final int foffset, final int flength,

    
   
2028
      final byte [] qualifier, final int qoffset, final int qlength)

    
   
2029
          throws IllegalArgumentException {

    
   
2030

   

    
   
2031
    long lLength = KeyValue.KEY_INFRASTRUCTURE_SIZE +

    
   
2032
                   row.length + flength + qlength +

    
   
2033
                   KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;

    
   
2034

   

    
   
2035
    if (lLength > Integer.MAX_VALUE) {

    
   
2036
      throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);

    
   
2037
    }

    
   
2038
    int iLength = (int) lLength;

    
   
2039
    if (buffer.length < iLength) {

    
   
2040
      throw new IllegalArgumentException("Buffer size " + buffer.length + " < " + iLength);

    
   
2041
    }

    
   
2042
    return new KeyValue(buffer,

    
   
2043
                        row, 0, row.length,

    
   
2044
                        family, foffset, flength,

    
   
2045
                        qualifier, qoffset, qlength,

    
   
2046
                        HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,

    
   
2047
                        null, 0, 0);

    
   
2048
  }

    
   
2049

   
1822
  /**
2050
  /**
1823
   * Create a KeyValue for the specified row, family and qualifier that would be
2051
   * Create a KeyValue for the specified row, family and qualifier that would be
1824
   * larger than or equal to all other possible KeyValues that have the same
2052
   * larger than or equal to all other possible KeyValues that have the same
1825
   * row, family, qualifier.
2053
   * row, family, qualifier.
1826
   * Used for reseeking.
2054
   * Used for reseeking.
[+20] [20] 428 lines
src/main/java/org/apache/hadoop/hbase/client/Result.java
Revision df0b3ef New Change
 
src/test/java/org/apache/hadoop/hbase/client/TestResult.java
Revision f9e29c2 New Change
 
  1. src/main/java/org/apache/hadoop/hbase/KeyValue.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/client/Result.java: Loading...
  3. src/test/java/org/apache/hadoop/hbase/client/TestResult.java: Loading...