cache.cpp

Go to the documentation of this file.

/* Copyright (c) 2011 Wildfire Games
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 * 
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include "precompiled.h"
#include "lib/sysdep/arch/x86_x64/cache.h"

#include "lib/bits.h"
#include "lib/alignment.h"
#include "lib/module_init.h"
#include "lib/sysdep/os_cpu.h"
#include "lib/sysdep/arch/x86_x64/x86_x64.h"

namespace x86_x64 {

static const size_t maxTLBs = 2*2*4;    // (level0, level1) x (D,I) x (4K, 2M, 4M, 1G)
static size_t numTLBs = 0;

static const size_t numCaches = x86_x64::Cache::maxLevels * 2 + maxTLBs;
static Cache caches[numCaches];


static void AddCache(const x86_x64::Cache& cache)
{
    ENSURE(cache.Validate());

    if(cache.type == x86_x64::Cache::kData || cache.type == x86_x64::Cache::kUnified)
        caches[L1D + cache.level-1] = cache;
    if(cache.type == x86_x64::Cache::kInstruction || cache.type == x86_x64::Cache::kUnified)
        caches[L1I + cache.level-1] = cache;
}


static void AddTLB(const x86_x64::Cache& tlb)
{
    ENSURE(tlb.Validate());
    ENSURE(tlb.level == 1 || tlb.level == 2);   // see maxTLBs

    ENSURE(numTLBs < maxTLBs);
    caches[TLB+numTLBs++] = tlb;
}


//-----------------------------------------------------------------------------
// AMD

// (Intel has subsequently added support for function 0x80000006, but
// only returns ECX, i.e. L2 information.)
namespace AMD
{

static x86_x64::Cache L1Cache(u32 reg, x86_x64::Cache::Type type)
{
    x86_x64::Cache cache;
    cache.Initialize(1, type);

    const size_t lineSize      = bits(reg,  0,  7);
    const size_t associativity = bits(reg, 16, 23); // 0 = reserved
    const size_t totalSize     = bits(reg, 24, 31)*KiB;
    if(lineSize != 0 && associativity != 0 && totalSize != 0)
    {
        cache.numEntries    = totalSize / lineSize;
        cache.entrySize     = lineSize;
        cache.associativity = associativity;
        cache.sharedBy      = 1;
    }
    return cache;
}

// applies to L2, L3 and TLB2
static const size_t associativityTable[16] =
{
    0, 1, 2, 0, 4, 0, 8, 0,
    16, 0, 32, 48, 64, 96, 128, x86_x64::Cache::fullyAssociative
};

static x86_x64::Cache L2Cache(u32 reg, x86_x64::Cache::Type type)
{
    x86_x64::Cache cache;
    cache.Initialize(2, type);

    const size_t lineSize         = bits(reg,  0,  7);
    const size_t idxAssociativity = bits(reg, 12, 15);  // 0 = disabled
    const size_t totalSize        = bits(reg, 16, 31)*KiB;
    if(lineSize != 0 && idxAssociativity != 0 && totalSize != 0)
    {
        cache.numEntries    = totalSize / lineSize;
        cache.entrySize     = lineSize;
        cache.associativity = associativityTable[idxAssociativity];
        cache.sharedBy      = 1;
    }
    return cache;
}

// (same as L2 except for the size)
static x86_x64::Cache L3Cache(u32 reg, x86_x64::Cache::Type type)
{
    x86_x64::Cache cache;
    cache.Initialize(3, type);

    const size_t lineSize         = bits(reg,  0,  7);
    const size_t idxAssociativity = bits(reg, 12, 15);  // 0 = disabled
    const size_t totalSize        = bits(reg, 18, 31)*512*KiB;  // (rounded down)
    // NB: some Athlon 64 X2 models have no L3 cache
    if(lineSize != 0 && idxAssociativity != 0 && totalSize != 0)
    {
        cache.numEntries    = totalSize / lineSize;
        cache.entrySize     = lineSize;
        cache.associativity = associativityTable[idxAssociativity];
        cache.sharedBy      = 1;
    }
    return cache;
}

static x86_x64::Cache TLB1(u32 reg, size_t bitOffset, size_t pageSize, x86_x64::Cache::Type type)
{
    x86_x64::Cache cache;
    cache.Initialize(1, type);

    const size_t numEntries    = bits(reg, bitOffset+0, bitOffset+ 7);
    const size_t associativity = bits(reg, bitOffset+8, bitOffset+15);  // 0 = reserved
    if(numEntries != 0 && associativity != 0)
    {
        cache.numEntries    = numEntries;
        cache.entrySize     = pageSize;
        cache.associativity = associativity;
        cache.sharedBy      = 1;
    }
    return cache;
}

static x86_x64::Cache TLB2(u32 reg, size_t bitOffset, size_t pageSize, x86_x64::Cache::Type type)
{
    x86_x64::Cache cache;
    cache.Initialize(2, type);

    const size_t numEntries       = bits(reg, bitOffset+ 0, bitOffset+11);
    const size_t idxAssociativity = bits(reg, bitOffset+12, bitOffset+15);  // 0 = disabled
    if(numEntries != 0 && idxAssociativity != 0)
    {
        cache.numEntries    = numEntries;
        cache.entrySize     = pageSize;
        cache.associativity = associativityTable[idxAssociativity];
        cache.sharedBy      = 1;
    }
    return cache;
}

static void AddTLB2Pair(u32 reg, size_t pageSize)
{
    x86_x64::Cache::Type type = x86_x64::Cache::kUnified;
    if(bits(reg, 16, 31) != 0)  // not unified
    {
        AddTLB(TLB2(reg, 16, pageSize, x86_x64::Cache::kData));
        type = x86_x64::Cache::kInstruction;
    }
    AddTLB(TLB2(reg, 0, pageSize, type));
}

// AMD reports maxCpuidIdFunction > 4 but consider functions 2..4 to be
// "reserved". cache characteristics are returned via ext. functions.
static void DetectCacheAndTLB()
{
    x86_x64::CpuidRegs regs = { 0 };

    regs.eax = 0x80000005;
    if(x86_x64::cpuid(&regs))
    {
        AddCache(L1Cache(regs.ecx, x86_x64::Cache::kData));
        AddCache(L1Cache(regs.edx, x86_x64::Cache::kInstruction));

        AddTLB(TLB1(regs.eax,  0, 2*MiB, x86_x64::Cache::kInstruction));
        AddTLB(TLB1(regs.eax, 16, 2*MiB, x86_x64::Cache::kData));
        AddTLB(TLB1(regs.ebx,  0, 4*KiB, x86_x64::Cache::kInstruction));
        AddTLB(TLB1(regs.ebx, 16, 4*KiB, x86_x64::Cache::kData));
    }

    regs.eax = 0x80000006;
    if(x86_x64::cpuid(&regs))
    {
        AddCache(L2Cache(regs.ecx, x86_x64::Cache::kUnified));
        AddCache(L3Cache(regs.edx, x86_x64::Cache::kUnified));

        AddTLB2Pair(regs.eax, 2*MiB);
        AddTLB2Pair(regs.ebx, 4*KiB);
    }
}

}   // namespace AMD


//-----------------------------------------------------------------------------
// CPUID.4

namespace CPUID4 {

static bool DetectCache()
{
    // note: level order is unspecified (see Intel AP-485)
    for(u32 count = 0; ; count++)
    {
        x86_x64::CpuidRegs regs = { 0 };
        regs.eax = 4;
        regs.ecx = count;
        if(!x86_x64::cpuid(&regs))
            return false;

        const x86_x64::Cache::Type type = (x86_x64::Cache::Type)bits(regs.eax, 0, 4);
        if(type == x86_x64::Cache::kNull)   // no more remaining
            break;

        const size_t level      = (size_t)bits(regs.eax, 5, 7);
        const size_t partitions = (size_t)bits(regs.ebx, 12, 21)+1;
        const size_t sets       = (size_t)bits(regs.ecx, 0, 31)+1;

        x86_x64::Cache cache;
        cache.Initialize(level, type);
        cache.entrySize     = (size_t)bits(regs.ebx,  0, 11)+1; // (yes, this also uses +1 encoding)
        cache.associativity = (size_t)bits(regs.ebx, 22, 31)+1;
        cache.sharedBy      = (size_t)bits(regs.eax, 14, 25)+1;
        cache.numEntries    = cache.associativity * partitions * sets;

        AddCache(cache);
    }

    return true;
}

}   // namespace CPUID4


//-----------------------------------------------------------------------------
// CPUID.2 (descriptors)

namespace CPUID2 {

typedef u8 Descriptor;
typedef std::vector<Descriptor> Descriptors;

static void AppendDescriptors(u32 reg, Descriptors& descriptors)
{
    if(IsBitSet(reg, 31))   // register contents are reserved
        return;
    for(int pos = 24; pos >= 0; pos -= 8)
    {
        const u8 descriptor = (u8)bits(reg, pos, pos+7);
        if(descriptor != 0)
            descriptors.push_back(descriptor);
    }
}


static Descriptors GetDescriptors()
{
    // ensure consistency by pinning to a CPU.
    // (don't use a hard-coded mask because process affinity may be restricted)
    const uintptr_t allProcessors = os_cpu_ProcessorMask();
    const uintptr_t firstProcessor = allProcessors & -intptr_t(allProcessors);
    const uintptr_t prevAffinityMask = os_cpu_SetThreadAffinityMask(firstProcessor);

    x86_x64::CpuidRegs regs = { 0 };
    regs.eax = 2;
    if(!x86_x64::cpuid(&regs))
        return Descriptors();

    Descriptors descriptors;
    size_t iterations = bits(regs.eax, 0, 7);
    for(;;) // abort mid-loop (invoke CPUID exactly <iterations> times)
    {
        AppendDescriptors(bits(regs.eax, 8, 31), descriptors);
        AppendDescriptors(regs.ebx, descriptors);
        AppendDescriptors(regs.ecx, descriptors);
        AppendDescriptors(regs.edx, descriptors);
        if(--iterations == 0)
            break;
        regs.eax = 2;
        const bool ok = x86_x64::cpuid(&regs);
        ENSURE(ok);
    }

    os_cpu_SetThreadAffinityMask(prevAffinityMask);

    return descriptors;
}


// note: the following cannot be moved into a function because
// ARRAY_SIZE's template argument must not reference a local type.

enum Flags
{
    // level (bits 0..1)
    L1 = 1,
    L2,
    L3,

    // type (bits 2..3)
    I   = 0x04, // instruction
    D   = 0x08, // data
    U   = I|D   // unified

    // largeSize (bits 4..31 with bits 0..3 zeroed): TLB entrySize or cache numEntries
};

// (there are > 100 descriptors, so we squeeze all fields into 8 bytes.)
struct Characteristics  // POD
{
    x86_x64::Cache::Type Type() const
    {
        switch(flags & U)
        {
        case D:
            return x86_x64::Cache::kData;
        case I:
            return x86_x64::Cache::kInstruction;
        case U:
            return x86_x64::Cache::kUnified;
        default:
            DEBUG_WARN_ERR(ERR::LOGIC);
            return x86_x64::Cache::kNull;
        }
    }

    size_t Level() const
    {
        const size_t level = flags & 3;
        ENSURE(level != 0);
        return level;
    }

    bool IsTLB() const
    {
        return smallSize >= 0;
    }

    size_t NumEntries() const
    {
        return IsTLB()? (size_t)smallSize : (flags & ~0xF);
    }

    size_t EntrySize() const
    {
        return IsTLB()? (flags & ~0xF) : (size_t)(-smallSize);
    }

    u8 descriptor;
    u8 associativity;
    i16 smallSize;  // negative cache entrySize or TLB numEntries
    u32 flags;  // level, type, largeSize
};

static const u8 F = x86_x64::Cache::fullyAssociative;

#define CACHE(descriptor, flags, totalSize, assoc, entrySize)  { descriptor, assoc, -entrySize, flags | ((totalSize)/(entrySize)) }
#define TLB(descriptor, flags, entrySize, assoc, numEntries) { descriptor, assoc, numEntries, flags | (entrySize) }

// (we need to include cache descriptors because early Pentium4 don't implement CPUID.4)
// references: [accessed 2011-02-26]
// AP485 http://www.intel.com/Assets/PDF/appnote/241618.pdf
// sdman http://www.intel.com/Assets/PDF/manual/253666.pdf
// sandp http://www.sandpile.org/ia32/cpuid.htm
// opsol http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/i86pc/os/cpuid.c
static const Characteristics characteristicsTable[] =
{
    TLB  (0x01, L1|I,   4*KiB,  4,  32),
    TLB  (0x02, L1|I,   4*MiB,  F,   2),
    TLB  (0x03, L1|D,   4*KiB,  4,  64),
    TLB  (0x04, L1|D,   4*MiB,  4,   8),
    TLB  (0x05, L1|D,   4*MiB,  4,  32),

    CACHE(0x06, L1|I,   8*KiB,  4,  32),
    CACHE(0x08, L1|I,  16*KiB,  4,  32),
    CACHE(0x09, L1|I,  32*KiB,  4,  64),
    CACHE(0x0A, L1|I,   8*KiB,  2,  32),

    TLB  (0x0B, L1|I,   4*MiB,  4,   4),

    CACHE(0x0C, L1|D,  16*KiB,  4,  32),
    CACHE(0x0D, L1|D,  16*KiB,  4,  64),    // opsol: 32B (would be redundant with 0x0C), AP485: 64B, sdman: 64B
    CACHE(0x0E, L1|D,  24*KiB,  6,  64),

    CACHE(0x21, L2|U, 256*KiB,  8,  64),

    CACHE(0x22, L3|U, 512*KiB,  4,  64),
    CACHE(0x23, L3|U,   1*MiB,  8,  64),
    CACHE(0x25, L3|U,   2*MiB,  8,  64),
    CACHE(0x29, L3|U,   4*MiB,  8,  64),

    CACHE(0x2c, L1|D,  32*KiB,  8,  64),

    CACHE(0x30, L1|I,  32*KiB,  8,  64),

    CACHE(0x39, L2|U, 128*KiB,  4,  64),
    CACHE(0x3A, L2|U, 192*KiB,  6,  64),
    CACHE(0x3B, L2|U, 128*KiB,  2,  64),
    CACHE(0x3C, L2|U, 256*KiB,  4,  64),
    CACHE(0x3D, L2|U, 384*KiB,  6,  64),
    CACHE(0x3E, L2|U, 512*KiB,  4,  64),
    CACHE(0x41, L2|U, 128*KiB,  4,  32),
    CACHE(0x42, L2|U, 256*KiB,  4,  32),
    CACHE(0x43, L2|U, 512*KiB,  4,  32),
    CACHE(0x44, L2|U,   1*MiB,  4,  32),
    CACHE(0x45, L2|U,   2*MiB,  4,  32),

    CACHE(0x46, L3|U,   4*MiB,  4,  64),
    CACHE(0x47, L3|U,   8*MiB,  8,  64),
    CACHE(0x48, L2|U,   3*MiB, 12,  64),
    CACHE(0x49, L2|U,   4*MiB, 16,  64),
    CACHE(0x49, L3|U,   4*MiB, 16,  64),
    CACHE(0x4A, L3|U,   6*MiB, 12,  64),
    CACHE(0x4B, L3|U,   8*MiB, 16,  64),
    CACHE(0x4C, L3|U,  12*MiB, 12,  64),
    CACHE(0x4D, L3|U,  16*MiB, 16,  64),
    CACHE(0x4E, L2|U,   6*MiB, 24,  64),

    TLB  (0x4F, L1|I,   4*KiB,  F,  32),    // sandp: unknown assoc, opsol: full, AP485: unspecified
    TLB  (0x50, L1|I,   4*KiB,  F,  64),
    TLB  (0x50, L1|I,   4*MiB,  F,  64),
    TLB  (0x50, L1|I,   2*MiB,  F,  64),
    TLB  (0x51, L1|I,   4*KiB,  F, 128),
    TLB  (0x51, L1|I,   4*MiB,  F, 128),
    TLB  (0x51, L1|I,   2*MiB,  F, 128),
    TLB  (0x52, L1|I,   4*KiB,  F, 256),
    TLB  (0x52, L1|I,   4*MiB,  F, 256),
    TLB  (0x52, L1|I,   2*MiB,  F, 256),
    TLB  (0x55, L1|I,   4*MiB,  F,   7),
    TLB  (0x55, L1|I,   2*MiB,  F,   7),

    TLB  (0x56, L1|D,   4*MiB,  4,  16),
    TLB  (0x57, L1|D,   4*KiB,  4,  16),
    TLB  (0x59, L1|D,   4*KiB,  F,  16),
    TLB  (0x5A, L1|D,   4*MiB,  4,  32),
    TLB  (0x5A, L1|D,   2*MiB,  4,  32),
    TLB  (0x5B, L1|D,   4*KiB,  F,  64),
    TLB  (0x5B, L1|D,   4*MiB,  F,  64),
    TLB  (0x5C, L1|D,   4*KiB,  F, 128),
    TLB  (0x5C, L1|D,   4*MiB,  F, 128),
    TLB  (0x5D, L1|D,   4*KiB,  F, 256),
    TLB  (0x5D, L1|D,   4*MiB,  F, 256),

    CACHE(0x60, L1|D,  16*KiB,  8,  64),
    TLB  (0x63, L1|D,   1*GiB,  4,   4),    // speculation
    CACHE(0x66, L1|D,   8*KiB,  4,  64),
    CACHE(0x67, L1|D,  16*KiB,  4,  64),
    CACHE(0x68, L1|D,  32*KiB,  4,  64),

    CACHE(0x70, L1|I,  12*KiB,  8,   1),
    CACHE(0x71, L1|I,  16*KiB,  8,   1),
    CACHE(0x72, L1|I,  32*KiB,  8,   1),
    CACHE(0x73, L1|I,  64*KiB,  8,   1),

    TLB  (0x76, L1|I,   4*MiB,  F,   8),    // AP485: internally inconsistent, sdman: TLB
    TLB  (0x76, L1|I,   2*MiB,  F,   8),

    CACHE(0x78, L2|U,   1*MiB,  4,  64),
    CACHE(0x79, L2|U, 128*KiB,  8,  64),
    CACHE(0x7A, L2|U, 256*KiB,  8,  64),
    CACHE(0x7B, L2|U, 512*KiB,  8,  64),
    CACHE(0x7C, L2|U,   1*MiB,  8,  64),
    CACHE(0x7D, L2|U,   2*MiB,  8,  64),
    CACHE(0x7F, L2|U, 512*KiB,  2,  64),

    CACHE(0x80, L2|U, 512*KiB,  8,  64),
    CACHE(0x82, L2|U, 256*KiB,  8,  32),
    CACHE(0x83, L2|U, 512*KiB,  8,  32),
    CACHE(0x84, L2|U,   1*MiB,  8,  32),
    CACHE(0x85, L2|U,   2*MiB,  8,  32),
    CACHE(0x86, L2|U, 512*KiB,  4,  64),
    CACHE(0x87, L2|U,   1*MiB,  8,  64),

    TLB  (0xB0, L1|I,   4*KiB,  4, 128),
    TLB  (0xB1, L1|I,   2*MiB,  4,   8),
    TLB  (0xB1, L1|I,   4*MiB,  4,   4),
    TLB  (0xB2, L1|I,   4*KiB,  4,  64),

    TLB  (0xB3, L1|D,   4*KiB,  4, 128),
    TLB  (0xB3, L1|D,   4*MiB,  4, 128),
    TLB  (0xB4, L1|D,   4*KiB,  4, 256),
    TLB  (0xB4, L1|D,   4*MiB,  4, 256),
    TLB  (0xB5, L1|I,   4*KiB,  4, 128),    // speculation
    TLB  (0xB6, L1|I,   4*KiB,  8, 128),    // http://software.intel.com/en-us/forums/topic/401012

    TLB  (0xBA, L1|D,   4*KiB,  4,  64),
    TLB  (0xC0, L1|D,   4*KiB,  4,   8),
    TLB  (0xC0, L1|D,   4*MiB,  4,   8),
    TLB  (0xC1, L2|U,   4*KiB,  8, 1024),   // http://software.intel.com/en-us/forums/topic/401012
    TLB  (0xC1, L2|U,   4*MiB,  8, 1024),
    TLB  (0xC1, L2|U,   2*MiB,  8, 1024),
    TLB  (0xCA, L2|U,   4*KiB,  4, 512),

    CACHE(0xD0, L3|U, 512*KiB,  4,  64),
    CACHE(0xD1, L3|U,   1*MiB,  4,  64),
    CACHE(0xD2, L3|U,   2*MiB,  4,  64),
    CACHE(0xD6, L3|U,   1*MiB,  8,  64),
    CACHE(0xD7, L3|U,   2*MiB,  8,  64),
    CACHE(0xD8, L3|U,   4*MiB,  8,  64),
    CACHE(0xDC, L3|U, 3*MiB/2, 12,  64),
    CACHE(0xDD, L3|U,   3*MiB, 12,  64),
    CACHE(0xDE, L3|U,   6*MiB, 12,  64),
    CACHE(0xE2, L3|U,   2*MiB, 16,  64),
    CACHE(0xE3, L3|U,   4*MiB, 16,  64),
    CACHE(0xE4, L3|U,   8*MiB, 16,  64),
    CACHE(0xEA, L3|U,  12*MiB, 24,  64),
    CACHE(0xEB, L3|U,  18*MiB, 24,  64),
    CACHE(0xEC, L3|U,  24*MiB, 24,  64),
};
#undef CACHE
#undef TLB

static const Characteristics* CharacteristicsFromDescriptor(Descriptor descriptor)
{
    // note: we can't use bsearch because characteristicsTable contains multiple
    // entries with the same descriptor.
    for(size_t i = 0; i < ARRAY_SIZE(characteristicsTable); i++)
    {
        const Characteristics& characteristics = characteristicsTable[i];
        if(characteristics.descriptor == descriptor)
            return &characteristics;
    }

    debug_printf(L"Unknown cache/TLB descriptor 0x%x\n", (unsigned int)descriptor);
    return 0;
}


enum DescriptorFlags
{
    SKIP_CACHE_DESCRIPTORS = 1,
    NO_LAST_LEVEL_CACHE    = 2,
    PREFETCH64             = 64,
    PREFETCH128            = 128
};

static bool HandleSpecialDescriptor(Descriptor descriptor, size_t& descriptorFlags)
{
    switch(descriptor)
    {
    case 0: // carries no information
        return true;

    case 0x40:
        descriptorFlags |= NO_LAST_LEVEL_CACHE;
        return true;

    case 0xF0:
        descriptorFlags |= PREFETCH64;
        return true;

    case 0xF1:
        descriptorFlags |= PREFETCH128;
        return true;

    case 0xFF:  // descriptors don't include caches (use CPUID.4 instead)
        descriptorFlags |= SKIP_CACHE_DESCRIPTORS;
        return true;

    default:
        return false;
    }
}


static void DetectCacheAndTLB(size_t& descriptorFlags)
{
    const Descriptors descriptors = GetDescriptors();
    for(Descriptors::const_iterator it = descriptors.begin(); it != descriptors.end(); ++it)
    {
        const Descriptor descriptor = *it;
        if(HandleSpecialDescriptor(descriptor, descriptorFlags))
            continue;

        const Characteristics* characteristics = CharacteristicsFromDescriptor(*it);
        if(!characteristics)
            continue;

        if((descriptorFlags & SKIP_CACHE_DESCRIPTORS) && !characteristics->IsTLB())
            continue;

        x86_x64::Cache cache;
        cache.Initialize(characteristics->Level(), characteristics->Type());
        cache.numEntries    = characteristics->NumEntries();
        cache.entrySize     = characteristics->EntrySize();
        cache.associativity = characteristics->associativity;
        cache.sharedBy      = 1;    // (safe default)
        if(characteristics->IsTLB())
            AddTLB(cache);
        else
            AddCache(cache);
    }
}

}   // namespace CPUID2


static Status DetectCacheAndTLB()
{
    // ensure all cache entries are initialized (DetectCache* might not set them all)
    for(size_t idxLevel = 0; idxLevel < x86_x64::Cache::maxLevels; idxLevel++)
    {
        caches[L1D+idxLevel].Initialize(idxLevel+1, x86_x64::Cache::kData);
        caches[L1I+idxLevel].Initialize(idxLevel+1, x86_x64::Cache::kInstruction);
    }

    if(x86_x64::Vendor() == x86_x64::VENDOR_AMD)
        AMD::DetectCacheAndTLB();
    else
    {
        size_t descriptorFlags = 0;
        if(CPUID4::DetectCache())   // success, ignore less reliable CPUID.2 cache information 
            descriptorFlags |= CPUID2::SKIP_CACHE_DESCRIPTORS;
        CPUID2::DetectCacheAndTLB(descriptorFlags);
    }

    // sanity checks
    for(size_t idxLevel = 0; idxLevel < x86_x64::Cache::maxLevels; idxLevel++)
    {
        ENSURE(caches[L1D+idxLevel].type == x86_x64::Cache::kData || caches[L1D+idxLevel].type == x86_x64::Cache::kUnified);
        ENSURE(caches[L1D+idxLevel].level == idxLevel+1);
        ENSURE(caches[L1D+idxLevel].Validate() == true);

        ENSURE(caches[L1I+idxLevel].type == x86_x64::Cache::kInstruction || caches[L1I+idxLevel].type == x86_x64::Cache::kUnified);
        ENSURE(caches[L1I+idxLevel].level == idxLevel+1);
        ENSURE(caches[L1I+idxLevel].Validate() == true);
    }
    for(size_t i = 0; i < numTLBs; i++)
        ENSURE(caches[TLB+i].Validate() == true);

    return INFO::OK;
}

const x86_x64::Cache* Caches(size_t idxCache)
{
    static ModuleInitState initState;
    ModuleInit(&initState, DetectCacheAndTLB);

    if(idxCache >= TLB+numTLBs)
        return 0;

    return &caches[idxCache];
}

}   // namespace x86_x64