Method tryEviction

memstore/host_memory_manager.go:406–461

tryEviction : try to trigger eviction once unManagedMem + managedMem > totalAssignedMem. This method will pop batches from the per column holder data structure, calculate global priority based on column metadata, then push the batch into a priority queue. Eviction will happen through all the populat

()

// decreases to a certain level. All failed eviction batches will be
// reinserted.
func (h *hostMemoryManager) tryEviction() {
	// Check if eviction should be triggered
	if (h.totalMemorySize - h.getManagedSpaceUsage() - h.getUnmanagedSpaceUsage()) < 0 {
		utils.GetLogger().Debugf("UnmanagedMem: %d + ManagedMem: %d is larger than totalMem: %d! Eviction is triggered.",
			h.getUnmanagedSpaceUsage(), h.getManagedSpaceUsage(), h.totalMemorySize)
		// Init all columnar priority batches.
		gpq := h.initialGlobalPriorityQueue()
		// Pop from globalPriorityQueueWithLock and do eviction
		for (h.totalMemorySize-h.getManagedSpaceUsage()-h.getUnmanagedSpaceUsage()) < 0 && !gpq.isEmpty() {
			globalPriorityItem := gpq.pop()

			batchPriority := globalPriorityItem.priority
			columnBatchInfos := globalPriorityItem.value
			columnIt := globalPriorityItem.it

			tableSchema, err := h.memStore.GetSchema(columnBatchInfos.table)

			tableSchema.RLock()
			preloadingDays := tableSchema.Schema.Columns[batchPriority.columnID].Config.PreloadingDays
			tableSchema.RUnlock()

			isPreloadingDays := isPreloadingBatch(batchPriority.batchID, preloadingDays)

			if isPreloadingDays {
				utils.GetReporter(columnBatchInfos.table, batchPriority.shardID).
					GetCounter(utils.PreloadingZoneEvicted).Inc(1)
				utils.GetLogger().With(
					"table", columnBatchInfos.table,
					"shard", batchPriority.shardID,
					"batch", batchPriority.batchID,
					"column", batchPriority.columnID,
				).Warn("Column in preloading zone is evicted")
			}

			ok, err := h.memStore.TryEvictBatchColumn(columnBatchInfos.table, batchPriority.shardID, int32(batchPriority.batchID), batchPriority.columnID)
			if ok {
				utils.GetLogger().Debugf("Successfully evict batch from memstore: table %s, shardID %d, batchID %d, columnID %d, size %d",
					columnBatchInfos.table, batchPriority.shardID, batchPriority.batchID, batchPriority.columnID, batchPriority.size)
			} else {
				utils.GetLogger().Debugf("Failed to evict batch from memstore: table %s, shardID %d, batchID %d, columnID %d, size %d, errors: %s",
					columnBatchInfos.table, batchPriority.shardID, batchPriority.batchID, batchPriority.columnID, batchPriority.size, err)
			}

			// Adding the corresponding next batch into priority queue.
			if columnIt.Next() {
				gpq.pushBatchIntoGlobalPriorityQueue(h, columnBatchInfos, batchPriority.columnID, columnIt)
			}
		}

		// Still cannot meet the memory constraints even after evictions.
		if h.totalMemorySize-h.getManagedSpaceUsage()-h.getUnmanagedSpaceUsage() < 0 {
			utils.GetRootReporter().GetCounter(utils.MemoryOverflow).Inc(1)
			utils.GetLogger().Warn("Still cannot meet the memory constraints even after evictions")
		}
	}
}

// pushBatchIntoGlobalPriorityQueue will generate a globalPriority object then