Thread safety with stormlib

[namreeb]

Just thought I'd share this.

For probably ten years now I've been using stormlib on Windows in a multi-threaded application. To my knowledge I have never encountered a problem. Recently when I tried the same code on Linux, I had all kinds of problems. After troubleshooting this for a while, I discovered that stormlib keeps an internal state associated with the file handles, which means it is not thread safe. I'm not sure why this was never a problem for me on Windows. I'm guessing that it has something to do with how the underlying file I/O works on Windows versus Linux.

At any rate, I was able to fix my problem on Linux by adding a mutex to prevent multiple threads from reading from the MPQs simultaneously. This slowed down my code by 300% or so, which I did not want to live with -- which is just stubbornness on my part since my application is pre-processing and execution time doesn't really matter anyway. I asked the stormlib author if opening multiple read-only handles to the MPQs, one handle per file per thread, would work, and he said that he thought it should. I tried it, and success!

Below is the code that resulted, which uses a thread-local singleton. It must be initialized with the proper data directory in each thread, which was easy for me to add to the function which creates my worker threads.

MpqManager.hpp:

#pragma once

#include  "utility/BinaryStream.hpp"

#include  <string>
#include  <vector>
#include  <unordered_map>

namespace parser
{
class MpqManager
{
    private:
        using HANDLE = void *;

        std::vector<HANDLE> MpqHandles;
        std::unordered_map<std::string, unsigned int> Maps;

        void LoadMpq(const std::string &filePath);

    public:
        void Initialize();
        void Initialize(const std::string &wowDir);

        utility::BinaryStream *OpenFile(const std::string &file);

        unsigned int GetMapId(const std::string &name);
};

extern thread_local MpqManager sMpqManager;
};

MpqManager.cpp:

#include  "MpqManager.hpp"
#include  "DBC.hpp"

#include  "utility/Exception.hpp"
#include  "StormLib.h"

#include  <vector>
#include  <sstream>
#include  <cstdint>
#include  <cctype>
#include  <algorithm>
#include  <unordered_map>
#include  <experimental/filesystem>
#include  <iostream>

namespace fs = std::experimental::filesystem;

namespace parser
{
thread_local MpqManager sMpqManager;

void MpqManager::LoadMpq(const std::string &filePath)
{
    HANDLE archive;
        
    if (!SFileOpenArchive(filePath.c_str(), 0, MPQ_OPEN_READ_ONLY, &archive))
        THROW("Could not open MPQ").ErrorCode();

    MpqHandles.push_back(archive);
}

void MpqManager::Initialize()
{
    Initialize(".");
}

// TODO examine how the retail clients determine MPQ loading order
void MpqManager::Initialize(const std::string &wowDir)
{
    auto const wowPath = fs::path(wowDir);

    std::vector<fs::directory_entry> directories;
    std::vector<fs::path> files;
    std::vector<fs::path> patches;

    for (auto i = fs::directory_iterator(wowPath); i != fs::directory_iterator(); ++i)
    {
        if (fs::is_directory(i->status()))
        {
            directories.push_back(*i);
            continue;
        }

        if (!fs::is_regular_file(i->status()))
            continue;

        auto path = i->path().string();
        std::transform(path.begin(), path.end(), path.begin(), ::tolower);

        if (path.find(".mpq") == std::string::npos)
            continue;

        if (path.find("wow-update") == std::string::npos)
            files.push_back(i->path());
        else
            patches.push_back(i->path());
    }

    if (files.empty() && patches.empty())
        THROW("Found no MPQs");

    std::sort(files.begin(), files.end());
    std::sort(patches.begin(), patches.end());

    // all locale directories should have files named lcLE\locale-lcLE.mpq and lcLE\patch-lcLE*.mpq
    for (auto const &dir : directories)
    {
        auto const dirString = dir.path().filename().string();

        if (dirString.length() != 4)
            continue;

        auto const localeMpq = wowPath / dirString / ("locale-" + dirString + ".MPQ");
        auto found = false;

        std::vector<fs::path> localePatches;
        fs::path firstPatch;
        for (auto i = fs::directory_iterator(dir); i != fs::directory_iterator(); ++i)
        {
            if (fs::equivalent(*i, localeMpq))
                found = true;

            auto const filename = i->path().filename().string();

            if (filename.find("patch-" + dirString + "-") != std::string::npos)
                localePatches.push_back(i->path());
            else if (filename.find("patch-" + dirString) != std::string::npos)
                firstPatch = i->path();
        }

        if (found)
        {
            files.push_back(localeMpq);
            if (!fs::is_empty(firstPatch))
                files.push_back(firstPatch);

            std::sort(localePatches.begin(), localePatches.end());
            std::copy(localePatches.cbegin(), localePatches.cend(), std::back_inserter(files));
        }
    }

    // the current belief is that the game uses files from mpqs in reverse alphabetical order
    // this still needs to be checked.
    std::reverse(std::begin(files), std::end(files));
    std::reverse(std::begin(patches), std::end(patches));

    for (auto const &file : files)
        LoadMpq(file.string());

    for (auto const &file : patches)
        for (auto const &handle : MpqHandles)
            if (!SFileOpenPatchArchive(handle, file.string().c_str(), "base", 0))
                THROW("Failed to apply patch").ErrorCode();

    DBC maps("DBFilesClient\\Map.dbc");

    for (auto i = 0u; i < maps.RecordCount(); ++i)
    {
        auto const map_name = maps.GetStringField(i, 1);
        std::string map_name_lower;
        std::transform(map_name.begin(), map_name.end(), std::back_inserter(map_name_lower), ::tolower);

        Maps[map_name_lower] = maps.GetField(i, 0);
    }
}

utility::BinaryStream *MpqManager::OpenFile(const std::string &file)
{
    if (MpqHandles.empty())
        THROW("MpqManager not initialized");

    for (auto const &handle : MpqHandles)
    {
        if (!SFileHasFile(handle, file.c_str()))
            continue;

        HANDLE fileHandle;
        if (!SFileOpenFileEx(handle, file.c_str(), SFILE_OPEN_FROM_MPQ, &fileHandle))
            THROW("Error in SFileOpenFileEx").ErrorCode();

        auto const fileSize = SFileGetFileSize(fileHandle, nullptr);

        if (!fileSize)
            continue;

        std::vector<std::uint8_t> inFileData(fileSize);

        if (!SFileReadFile(fileHandle, &inFileData[0], static_cast<DWORD>(inFileData.size()), nullptr, nullptr))
        {
            SFileCloseFile(fileHandle);
            THROW("Error in SFileReadFile").ErrorCode();
        }

        SFileCloseFile(fileHandle);

        return new utility::BinaryStream(inFileData);
    }

    return nullptr;
}

unsigned int MpqManager::GetMapId(const std::string &name)
{
    std::string nameLower;
    std::transform(name.begin(), name.end(), std::back_inserter(nameLower), ::tolower);

    auto const i = Maps.find(nameLower);

    if (i == Maps.end())
        THROW("Map ID for " + name + " not found");

    return i->second;
}
}