Method TestWriteErrorBlacklist

be/src/runtime/bufferpool/buffer-pool-test.cc:1741–1871

Test that scratch devices are blacklisted after a write error. The query that encountered the write error should not allocate more pages on that device, but existing pages on the device will remain in use and future queries will use the device.

// encountered the write error should not allocate more pages on that device, but
// existing pages on the device will remain in use and future queries will use the device.
void BufferPoolTest::TestWriteErrorBlacklist(
    const string& compression, bool punch_holes) {
  // Set up two file groups with two temporary dirs.
  vector<string> tmp_dirs = InitTmpFileMgr(2, compression, punch_holes);
  // Simulate two concurrent queries.
  const int TOTAL_QUERIES = 3;
  const int INITIAL_QUERIES = 2;
  const int MAX_NUM_PAGES = 6;
  const int PAGES_PER_QUERY = MAX_NUM_PAGES / TOTAL_QUERIES;
  const int64_t TOTAL_MEM = MAX_NUM_PAGES * TEST_BUFFER_LEN;
  const int64_t MEM_PER_QUERY = PAGES_PER_QUERY * TEST_BUFFER_LEN;
  BufferPool pool(test_env_->metrics(), TEST_BUFFER_LEN, TOTAL_MEM, TOTAL_MEM);
  global_reservations_.InitRootTracker(NewProfile(), TOTAL_MEM);
  vector<TmpFileGroup*> file_groups;
  vector<ClientHandle> clients(TOTAL_QUERIES);
  for (int i = 0; i < INITIAL_QUERIES; ++i) {
    file_groups.push_back(NewFileGroup());
    ASSERT_OK(pool.RegisterClient("test client", file_groups[i], &global_reservations_,
        nullptr, MEM_PER_QUERY, NewProfile(), &clients[i]));
    ASSERT_TRUE(clients[i].IncreaseReservation(MEM_PER_QUERY));
  }

  // Allocate files for all 2x2 combinations by unpinning pages.
  vector<vector<PageHandle>> pages(TOTAL_QUERIES);
  for (int i = 0; i < INITIAL_QUERIES; ++i) {
    CreatePages(&pool, &clients[i], TEST_BUFFER_LEN, MEM_PER_QUERY, &pages[i]);
    WriteData(pages[i], 0);
    UnpinAll(&pool, &clients[i], &pages[i]);
    for (int j = 0; j < PAGES_PER_QUERY; ++j) {
      LOG(INFO) << "Manager " << i << " Block " << j << " backed by file "
                << TmpFilePath(&pages[i][j]);
    }
  }
  for (int i = 0; i < INITIAL_QUERIES; ++i) WaitForAllWrites(&clients[i]);
  const int ERROR_QUERY = 0;
  const int NO_ERROR_QUERY = 1;
  const string& error_dir = tmp_dirs[0];
  const string& good_dir = tmp_dirs[1];
  // Delete one file from first scratch dir for first query to trigger an error.
  PageHandle* error_page = FindPageInDir(pages[ERROR_QUERY], error_dir);
  ASSERT_TRUE(error_page != NULL)
      << TmpFilePaths(pages[ERROR_QUERY]) << " not in " << DumpScratchDir(error_dir);
  const string& error_file_path = TmpFilePath(error_page);
  for (int i = 0; i < INITIAL_QUERIES; ++i) {
    ASSERT_OK(PinAll(&pool, &clients[i], &pages[i]));
  }
  DisableBackingFile(error_file_path);
  for (int i = 0; i < INITIAL_QUERIES; ++i) UnpinAll(&pool, &clients[i], &pages[i]);

  // At least one write should hit an error, but it should be recoverable.
  for (int i = 0; i < INITIAL_QUERIES; ++i) WaitForAllWrites(&clients[i]);

  // Both clients should still be usable - test the API.
  for (int i = 0; i < INITIAL_QUERIES; ++i) {
    ASSERT_OK(PinAll(&pool, &clients[i], &pages[i]));
    VerifyData(pages[i], 0);
    UnpinAll(&pool, &clients[i], &pages[i]);
    ASSERT_OK(AllocateAndFree(&pool, &clients[i], TEST_BUFFER_LEN));