Need help parsing mpq war3.mpq C# Unity

[gambikules]

Hello. Need help . My goal is parsing mpq of my war3.mpq from my original Warcraft3 no patches no Addons . In unity .

Not JASS but c#

using System;
using System.IO;
using System.Text;
using System.IO.Compression;

public class WC3MPQ : IDisposable
{
    private Stream stream;
    private BinaryReader br;

    private uint hashOffset;
    private uint blockOffset;

    private int hashCount;
    private int blockCount;

    private MPQHashEntry[] hashTable;
    private MPQBlockEntry[] blockTable;

    private long baseOffset;

    public struct MPQHashEntry
    {
        public uint NameHashA;
        public uint NameHashB;
        public ushort Locale;
        public ushort Platform;
        public uint BlockIndex;
    }

    public struct MPQBlockEntry
    {
        public uint Offset;
        public uint CompressedSize;
        public uint Size;
        public uint Flags;
    }

    public WC3MPQ(Stream s)
    {
        stream = s;
        br = new BinaryReader(stream);

        SeekHeader();
        ReadHeader();
        ReadTables();
    }
    long FindMpqHeader()
    {
        byte[] sig = { 0x4D, 0x50, 0x51, 0x1A };
        byte[] buf = new byte[4];

        for (long i = 0; i < stream.Length - 4; i++)
        {
            stream.Position = i;
            stream.Read(buf, 0, 4);

            if (buf[0] == sig[0] &&
                buf[1] == sig[1] &&
                buf[2] == sig[2] &&
                buf[3] == sig[3])
            {
                return i;
            }
        }

        throw new Exception("MPQ header not found");
    }
    // ================= HEADER =================
    void SeekHeader()
    {
        long mpqBase = FindMpqHeader();

        baseOffset = mpqBase;

        stream.Position = baseOffset;

        uint sig = br.ReadUInt32();
        if (sig != 0x1A51504D)
            throw new Exception("Invalid MPQ header");

        uint headerSize = br.ReadUInt32();
        uint archiveSize = br.ReadUInt32();
        ushort formatVersion = br.ReadUInt16();
        ushort sectorShift = br.ReadUInt16();

        hashOffset = br.ReadUInt32();
        blockOffset = br.ReadUInt32();
        hashCount = (int)br.ReadUInt32();
        blockCount = (int)br.ReadUInt32();

        if (hashCount <= 0 || blockCount <= 0)
            throw new Exception("Invalid MPQ table sizes");
    }
    void ReadHeader()
    {

        stream.Position = baseOffset;

        uint sig = br.ReadUInt32(); // MPQ
        uint headerSize = br.ReadUInt32();
        uint archiveSize = br.ReadUInt32();

        ushort formatVersion = br.ReadUInt16();
        ushort sectorShift = br.ReadUInt16();

        hashOffset = br.ReadUInt32();
        blockOffset = br.ReadUInt32();

        hashCount = (int)br.ReadUInt32();
        blockCount = (int)br.ReadUInt32();

        UnityEngine.Debug.Log("headerSize" + headerSize);
        UnityEngine.Debug.Log("archiveSize" + archiveSize);
        UnityEngine.Debug.Log("hashOffset" + hashOffset);
        UnityEngine.Debug.Log("blockOffset" + blockOffset);
        UnityEngine.Debug.Log("hashCount" + hashCount);
        UnityEngine.Debug.Log("blockCount" + blockCount);
 
    }

    // ================= TABLES =================
    void ReadTables()
    {
        hashTable = new MPQHashEntry[hashCount];
        blockTable = new MPQBlockEntry[blockCount];

        // HASH TABLE
        byte[] hashBuffer = new byte[hashCount * 16];

        stream.Position = baseOffset + hashOffset;
        stream.Read(hashBuffer, 0, hashBuffer.Length);
        UnityEngine.Debug.Log(BitConverter.ToUInt32(hashBuffer, 0).ToString("X8"));
        UnityEngine.Debug.Log(BitConverter.ToUInt32(hashBuffer, 4).ToString("X8"));
        UnityEngine.Debug.Log(BitConverter.ToUInt32(hashBuffer, 12).ToString("X8"));
        Decrypt(hashBuffer, GetKey("(hash table)"));

        for (int i = 0; i < hashCount; i++)
        {
            int o = i * 16;

            hashTable[I].NameHashA = BitConverter.ToUInt32(hashBuffer, o);
            hashTable[I].NameHashB = BitConverter.ToUInt32(hashBuffer, o + 4);
            hashTable[I].Locale = BitConverter.ToUInt16(hashBuffer, o + 8);
            hashTable[I].Platform = BitConverter.ToUInt16(hashBuffer, o + 10);
            hashTable[I].BlockIndex = BitConverter.ToUInt32(hashBuffer, o + 12);
        }

        // BLOCK TABLE
        byte[] blockBuffer = new byte[blockCount * 16];

        stream.Position = baseOffset + blockOffset;
        stream.Read(blockBuffer, 0, blockBuffer.Length);

        Decrypt(blockBuffer, GetKey("(block table)"));

        for (int i = 0; i < blockCount; i++)
        {
            int o = i * 16;

            blockTable[I].Offset = BitConverter.ToUInt32(blockBuffer, o);
            blockTable[I].CompressedSize = BitConverter.ToUInt32(blockBuffer, o + 4);
            blockTable[I].Size = BitConverter.ToUInt32(blockBuffer, o + 8);
            blockTable[I].Flags = BitConverter.ToUInt32(blockBuffer, o + 12);
        }
    }
    uint GetKey(string str)
    {
        return HashInternal(str, 0x300, 0);
    }
    public bool Exists(string name)
    {
        return FindFile(name) != 0xFFFFFFFF;
    }
    // ================= FILE READ =================
    public byte[] ReadFile(string name)
    {
        uint index = FindFile(name);
        if (index == 0xFFFFFFFF)
            throw new Exception("File not found: " + name);

        var b = blockTable[index];

        stream.Position = baseOffset + b.Offset;

        byte[] data = br.ReadBytes((int)b.CompressedSize);

        if ((b.Flags & 0x00000200) == 0)
            return data;

        return Decompress(data);
    }

    // ================= FIND FILE =================
    public uint FindFile(string name)
    {
        uint hashA = HashA(name);
        uint hashB = HashB(name);

        uint pos = hashA % (uint)hashTable.Length;

        for (int i = 0; i < hashTable.Length; i++)
        {
            var e = hashTable[pos];

            if (e.BlockIndex == 0xFFFFFFFF)
                return 0xFFFFFFFF;

            if (e.BlockIndex != 0xFFFFFFFE &&
                e.NameHashA == hashA &&
                e.NameHashB == hashB)
            {
                return e.BlockIndex;
            }

            pos = (pos + 1) % (uint)hashTable.Length;
        }

        return 0xFFFFFFFF;
    }

    // ================= HASH =================
    public static uint[] cryptTable = InitCrypt();

    static uint[] InitCrypt()
    {
        uint[] table = new uint[0x500];
        uint seed = 0x00100001;

        for (uint i = 0; i < 0x100; i++)
        {
            for (uint j = i, k = 0; k < 5; k++, j += 0x100)
            {
                seed = (seed * 125 + 3) % 0x2AAAAB;
                uint t1 = (seed & 0xFFFF) << 16;

                seed = (seed * 125 + 3) % 0x2AAAAB;
                uint t2 = seed & 0xFFFF;

                table[j] = t1 | t2;
            }
        }

        return table;
    }

    uint Hash(string s)
    {
        return HashInternal(s, 0x300, 0);
    }

    uint HashA(string s)
    {
        return HashInternal(s, 0x7FED7FED, 0xEEEEEEEE);
    }

    uint HashB(string s)
    {
        return HashInternal(s, 0xEEEEEEEE, 0x7FED7FED);
    }

    uint HashInternal(string s, uint seed1, uint seed2)
    {
        s = s.Replace('/', '\\').ToUpperInvariant();

        byte[] bytes = Encoding.ASCII.GetBytes(s);

        foreach (byte c in bytes)
        {
            uint val = cryptTable[0x400 + c];

            seed1 = val ^ (seed1 + seed2);
            seed2 = c + seed1 + seed2 + (seed2 << 5) + 3;
        }

        return seed1;
    }

    // ================= DECRYPT =================
    void Decrypt(byte[] data, uint key)
    {
        uint seed1 = key;
        uint seed2 = 0xEEEEEEEE;

        for (int i = 0; i < data.Length; i += 4)
        {
            // update seed2
            seed2 += cryptTable[0x400 + (seed1 & 0xFF)];

            // read little-endian dword
            uint value =
                (uint)data
[I]                | ((uint)data[i + 1] << 8)
                | ((uint)data[i + 2] << 16)
                | ((uint)data[i + 3] << 24);

            // decrypt
            value ^= (seed1 + seed2);

            // update seed1 (IMPORTANT: 32-bit wrap)
            seed1 = unchecked(
                ((~seed1 << 21) + 0x11111111) |
                (seed1 >> 11)
            );

            // update seed2 (IMPORTANT: based on decrypted value)
            seed2 = value + seed2 + (seed2 << 5) + 3;

            // write back
            data[I] = (byte)(value);
            data[i + 1] = (byte)(value >> 8);
            data[i + 2] = (byte)(value >> 16);
            data[i + 3] = (byte)(value >> 24);
        }
    }

    // ================= DECOMPRESS =================
    byte[] Decompress(byte[] input)
    {
        if (input.Length > 0 && (input[0] == 0x02 || input[0] == 0x08))
        {
            byte[] tmp = new byte[input.Length - 1];
            Buffer.BlockCopy(input, 1, tmp, 0, tmp.Length);
            input = tmp;
        }

        using (var ms = new MemoryStream(input))
        using (var output = new MemoryStream())
        {
            using (var ds = new DeflateStream(ms, CompressionMode.Decompress))
            {
                ds.CopyTo(output);
            }

            return output.ToArray();
        }
    }

    public void Dispose()
    {
        br?.Dispose();
        stream?.Dispose();
    }
}[/I][/I][/I][/I][/I][/I][/I][/I][/I][/I][/I]

Problem are bad offsets for hashOffset and blockOffset. I am reading for wrong blocks ...

headerSize = 32;
archiveSize = 447331787;

formatVersion = 0
sectorShift = 3

hashOffset = 446635339
blockOffset = 447159627;

hashCount = 32768;
blockCount = 10760;

Same in Ladik's MPQ Editor.

stream.Position = hashOffset; baseOffset is 0

UnityEngine.Debug.Log(BitConverter.ToUInt32(hashBuffer, 0));
UnityEngine.Debug.Log(BitConverter.ToUInt32(hashBuffer, 4);
UnityEngine.Debug.Log(BitConverter.ToUInt32(hashBuffer, 12);

give me NameHashA = 311592236 NameHashB = 1331685488 BlockIndex = 2815808023

completely absurde BlockIndex. problem is in ReadTables() stream.Position and/or stream.Read() wrong BlockIndex after Decrypt or before

----------------------------------------
OK finally wrong HashString ... should be

seed1 = cryptTable[(type << 8) + ch] ^ (seed1 + seed2);
seed2 = (uint)(ch + seed1 + seed2 + (seed2 << 5) + 3);
now i have true BlockIndex and i can found Footmax.mdx and Footman.blp and maps

 

[Wareditor]

Not sure if you still need help or not.

 

[gambikules]

Parsing is good now. I can extract everything. Now i m stack at blp to jpg or png. BLP1 like footman.blp or other model textures

I cant convert to jpg. Cause its not rgb colors but compressed jpeg jifif. I cant using jpeg dll or stormlib for this. I m working on partial port WC3 to ps vita. Very limited. I have no idea how convert jpeg inside BLP1 to rgb colors.
Attached image is a result of just saving BLP bytes + jpeg header to the file .jpg...

 

[Retera]

So for me when I use Warcraft 3 rewrite in Java to simulate the game on other platforms or in other ways, it's obviously Java not C# and then instead of Unity it is loading on LibGDX game engine. But to make that work I do BLP parsing using the blp-iio-plugin from @Dr Super Good which I believe has all the knowledge you're looking for.

Here is a video (this is a video of LibGDX game engine, not Warcraft III !!!)

The code shown in the video is here:

GitHub - Retera/WarsmashModEngine: An emulation engine to improve Warcraft III modding

An emulation engine to improve Warcraft III modding - Retera/WarsmashModEngine

github.com

The BLP parser in that game system is here:

GitHub - Retera/blp-iio-plugin: A Java Image IO plugin for BLP texture files used in games like Warcraft III and World of Warcraft.

A Java Image IO plugin for BLP texture files used in games like Warcraft III and World of Warcraft. - Retera/blp-iio-plugin

github.com

which is forked from here:

GitHub - DrSuperGood/blp-iio-plugin: A Java Image IO plugin for BLP texture files used in games like Warcraft III and World of Warcraft.

A Java Image IO plugin for BLP texture files used in games like Warcraft III and World of Warcraft. - DrSuperGood/blp-iio-plugin

github.com

In my opinion, another good BLP parser is here:

mdx-m3-viewer/src/parsers/blp at master · flowtsohg/mdx-m3-viewer

A WebGL viewer for MDX and M3 files used by the games Warcraft 3 and Starcraft 2 respectively. - flowtsohg/mdx-m3-viewer

github.com

That one is used by the Hive Workshop when it does the "View in 3D" previous of models.

You could also take a look at the one from HiveWE, the open source world editor from Hive:

HiveWE/src/file_formats/blp.ixx at main · stijnherfst/HiveWE

A Warcraft III world editor. Contribute to stijnherfst/HiveWE development by creating an account on GitHub.

github.com

 

[gambikules]

So for me when I use Warcraft 3 rewrite in Java to simulate the game on other platforms or in other ways, it's obviously Java not C# and then instead of Unity it is loading on LibGDX game engine. But to make that work I do BLP parsing using the blp-iio-plugin from @Dr Super Good which I believe has all the knowledge you're looking for.

Here is a video (this is a video of LibGDX game engine, not Warcraft III !!!)

Thank you. Useful information.
This looks like exactly what I need.

 

[Dr Super Good]

Parsing is good now. I can extract everything. Now i m stack at blp to jpg or png. BLP1 like footman.blp or other model texturesView attachment 591760 I cant convert to jpg. Cause its not rgb colors but compressed jpeg jifif. I cant using jpeg dll or stormlib for this. I m working on partial port WC3 to ps vita. Very limited. I have no idea how convert jpeg inside BLP1 to rgb colors.
Attached image is a result of just saving BLP bytes + jpeg header to the file .jpg...

They are already RGB colours. More specifically they are always BGRA formatted JPEG images, even if no alpha is specified.

The problem you are running into is because your JPEG library is being a little bit too user friendly and automatically handling YCrCb conversion, since that is typically how one is meant to use JPEG images. A typical JPEG image is a YCrCb image, where as Warcraft III uses BGRA JPEG images. You might be able to hack around this by implementing your own inverse YCrCb conversion step, that should be able to regenerate the original JPEG images within +/- 1 intensity per component (due to rounding error).

A properly written JPEG library should allow you to directly extract the stored components without any implicit colour conversion happening. Java's built in library supports this by exporting as a raster. The old intel imaging libraries that classic WC3 used also supported this, which is the how and likely why WC3 uses such non-standard JPEG images.

Unless you require your PS Vita port to use the original classic files, then I would personally recommend rebundling the textures into a more Vita friendly format. This is what modern Reforged has done, where they ported the old BLP files into much more widely compatible DDS textures, although this process was imperfect (quality issues). The most efficient solution might be to store just the full resolution image in a standard, modern, lossy file format and then dynamically compute the mipmaps as most (all?) textures had continuous mipmaps. Otherwise if the Vita has its own texture file format (likely does?) then port the textures to that.

If you want to perfectly load WC3 BLP files, then you might need to write your own JPEG library. If you are using a third party open source JPEG library (and not an official one from Sony) then you might be able to modify that code rather than writing from scratch. What would need to be done is to disable unwanted colour space conversion, and to make sure it parses a 4th component plane as alpha (most do anyway, but some might just stop at 3 planes). The output will be BGRA image data, which might need the Blue and Red components swapped to be more friendly with some libraries and hardware. Modern graphics libraries should still be able to handle BGRA texture data natively.

 

[Drake53]

These kind of threads are always funny to me, because I wonder if my project is somehow not discoverable enough, or if the guy just went straight to re-inventing the wheel without looking for existing solutions.
Anyway, here's my repo, it has mpq, blp, and more, all in C#: GitHub - Drake53/War3Net: The complete .NET toolkit for Warcraft III modding.