ITTestLargeVector.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.arrow.vector;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.holders.NullableDecimalHolder;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Integration test for a vector with a large (more than 2GB) {@link
 * org.apache.arrow.memory.ArrowBuf} as the data buffer. To run this test, please make sure there is
 * at least 4GB free memory in the system.
 */
public class ITTestLargeVector {
  private static final Logger logger = LoggerFactory.getLogger(ITTestLargeVector.class);

  @Test
  public void testLargeLongVector() {
    logger.trace("Testing large big int vector.");

    final long bufSize = 4 * 1024 * 1024 * 1024L;
    final int vecLength = (int) (bufSize / BigIntVector.TYPE_WIDTH);

    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
        BigIntVector largeVec = new BigIntVector("vec", allocator)) {
      largeVec.allocateNew(vecLength);

      logger.trace("Successfully allocated a vector with capacity {}", vecLength);

      for (int i = 0; i < vecLength; i++) {
        largeVec.set(i, i * 10L);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully written {} values", i + 1);
        }
      }
      logger.trace("Successfully written {} values", vecLength);

      for (int i = 0; i < vecLength; i++) {
        long val = largeVec.get(i);
        assertEquals(i * 10L, val);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully read {} values", i + 1);
        }
      }
      logger.trace("Successfully read {} values", vecLength);
    }
    logger.trace("Successfully released the large vector.");
  }

  @Test
  public void testLargeIntVector() {
    logger.trace("Testing large int vector.");

    final long bufSize = 4 * 1024 * 1024 * 1024L;
    final int vecLength = (int) (bufSize / IntVector.TYPE_WIDTH);

    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
        IntVector largeVec = new IntVector("vec", allocator)) {
      largeVec.allocateNew(vecLength);

      logger.trace("Successfully allocated a vector with capacity {}", vecLength);

      for (int i = 0; i < vecLength; i++) {
        largeVec.set(i, i);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully written {} values", i + 1);
        }
      }
      logger.trace("Successfully written {} values", vecLength);

      for (int i = 0; i < vecLength; i++) {
        long val = largeVec.get(i);
        assertEquals(i, val);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully read {} values", i + 1);
        }
      }
      logger.trace("Successfully read {} values", vecLength);
    }
    logger.trace("Successfully released the large vector.");
  }

  @Test
  public void testLargeDecimalVector() {
    logger.trace("Testing large decimal vector.");

    final long bufSize = 4 * 1024 * 1024 * 1024L;
    final int vecLength = (int) (bufSize / DecimalVector.TYPE_WIDTH);

    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
        DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 0)) {
      largeVec.allocateNew(vecLength);

      logger.trace("Successfully allocated a vector with capacity {}", vecLength);

      for (int i = 0; i < vecLength; i++) {
        largeVec.set(i, 0);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully written {} values", i + 1);
        }
      }
      logger.trace("Successfully written {} values", vecLength);

      for (int i = 0; i < vecLength; i++) {
        ArrowBuf buf = largeVec.get(i);
        assertEquals(DecimalVector.TYPE_WIDTH, buf.capacity());
        assertEquals(0, buf.getLong(0));
        assertEquals(0, buf.getLong(8));

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully read {} values", i + 1);
        }
      }
      logger.trace("Successfully read {} values", vecLength);

      // try setting values with a large offset in the buffer
      largeVec.set(vecLength - 1, 12345L);
      assertEquals(12345L, largeVec.getObject(vecLength - 1).longValue());

      NullableDecimalHolder holder = new NullableDecimalHolder();
      holder.buffer = largeVec.valueBuffer;
      holder.isSet = 1;
      holder.start = (long) (vecLength - 1) * largeVec.getTypeWidth();
      assertTrue(holder.start > Integer.MAX_VALUE);
      largeVec.set(0, holder);

      BigDecimal decimal = largeVec.getObject(0);
      assertEquals(12345L, decimal.longValue());

      logger.trace("Successfully setting values from large offsets");
    }
    logger.trace("Successfully released the large vector.");
  }

  @Test
  public void testLargeFixedSizeBinaryVector() {
    logger.trace("Testing large fixed size binary vector.");

    final long bufSize = 4 * 1024 * 1024 * 1024L;
    final int typeWidth = 8;
    final int vecLength = (int) (bufSize / typeWidth);

    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
        FixedSizeBinaryVector largeVec = new FixedSizeBinaryVector("vec", allocator, typeWidth)) {
      largeVec.allocateNew(vecLength);

      logger.trace("Successfully allocated a vector with capacity {}", vecLength);

      byte[] value = new byte[] {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'};
      for (int i = 0; i < vecLength; i++) {
        largeVec.set(i, value);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully written {} values", i + 1);
        }
      }
      logger.trace("Successfully written {} values", vecLength);

      for (int i = 0; i < vecLength; i++) {
        byte[] buf = largeVec.get(i);
        assertEquals(typeWidth, buf.length);
        assertArrayEquals(buf, value);

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully read {} values", i + 1);
        }
      }
      logger.trace("Successfully read {} values", vecLength);
    }
    logger.trace("Successfully released the large vector.");
  }

  @Test
  public void testLargeVarCharVector() {
    logger.trace("Testing large var char vector.");

    final long bufSize = 4 * 1024 * 1024 * 1024L;
    final int vecLength = (int) (bufSize / BaseVariableWidthVector.OFFSET_WIDTH);
    final String strElement = "a";

    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
        VarCharVector largeVec = new VarCharVector("vec", allocator)) {
      largeVec.allocateNew(vecLength);

      logger.trace("Successfully allocated a vector with capacity " + vecLength);

      for (int i = 0; i < vecLength; i++) {
        largeVec.setSafe(i, strElement.getBytes(StandardCharsets.UTF_8));

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully written " + (i + 1) + " values");
        }
      }
      largeVec.setValueCount(vecLength);
      assertTrue(largeVec.getOffsetBuffer().readableBytes() > Integer.MAX_VALUE);
      assertTrue(largeVec.getDataBuffer().readableBytes() < Integer.MAX_VALUE);
      logger.trace("Successfully written " + vecLength + " values");

      for (int i = 0; i < vecLength; i++) {
        byte[] val = largeVec.get(i);
        assertEquals(strElement, new String(val, StandardCharsets.UTF_8));

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully read " + (i + 1) + " values");
        }
      }
      logger.trace("Successfully read " + vecLength + " values");
    }
    logger.trace("Successfully released the large vector.");
  }

  @Test
  public void testLargeLargeVarCharVector() {
    logger.trace("Testing large large var char vector.");

    final long bufSize = 4 * 1024 * 1024 * 1024L;
    final int vecLength = (int) (bufSize / BaseLargeVariableWidthVector.OFFSET_WIDTH);
    final String strElement = "9876543210";

    try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
        LargeVarCharVector largeVec = new LargeVarCharVector("vec", allocator)) {
      largeVec.allocateNew(vecLength);

      logger.trace("Successfully allocated a vector with capacity " + vecLength);

      for (int i = 0; i < vecLength; i++) {
        largeVec.setSafe(i, strElement.getBytes(StandardCharsets.UTF_8));

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully written " + (i + 1) + " values");
        }
      }
      largeVec.setValueCount(vecLength);
      assertTrue(largeVec.getOffsetBuffer().readableBytes() > Integer.MAX_VALUE);
      assertTrue(largeVec.getDataBuffer().readableBytes() > Integer.MAX_VALUE);
      logger.trace("Successfully written " + vecLength + " values");

      for (int i = 0; i < vecLength; i++) {
        byte[] val = largeVec.get(i);
        assertEquals(strElement, new String(val, StandardCharsets.UTF_8));

        if ((i + 1) % 10000 == 0) {
          logger.trace("Successfully read " + (i + 1) + " values");
        }
      }
      logger.trace("Successfully read " + vecLength + " values");
    }
    logger.trace("Successfully released the large vector.");
  }
}